You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2019/02/24 08:15:12 UTC
[hive] branch master updated: HIVE-21307: Need to set
GzipJSONMessageEncoder as default config for EVENT_MESSAGE_FACTORY (Sankar
Hariappan, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2daaed7 HIVE-21307: Need to set GzipJSONMessageEncoder as default config for EVENT_MESSAGE_FACTORY (Sankar Hariappan, reviewed by Mahesh Kumar Behera)
2daaed7 is described below
commit 2daaed73de09b4b7987fd2d682140d7ee5637640
Author: Sankar Hariappan <sa...@apache.org>
AuthorDate: Sun Feb 24 13:44:53 2019 +0530
HIVE-21307: Need to set GzipJSONMessageEncoder as default config for EVENT_MESSAGE_FACTORY (Sankar Hariappan, reviewed by Mahesh Kumar Behera)
Signed-off-by: Sankar Hariappan <sa...@apache.org>
---
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
.../test/java/org/apache/hive/hcatalog/api/TestHCatClient.java | 5 ++++-
.../apache/hive/hcatalog/api/TestHCatClientNotification.java | 6 ++++--
.../hive/hcatalog/listener/TestDbNotificationListener.java | 1 +
.../hive/ql/parse/TestReplAcidTablesWithJsonMessage.java | 9 ++++++++-
.../ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java | 10 ++++++++--
.../TestReplIncrementalLoadAcidTablesWithJsonMessage.java | 10 ++++++++--
.../hive/ql/parse/TestReplTableMigrationWithJsonFormat.java | 10 ++++++++--
.../hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java | 9 ++++++++-
.../org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 2 +-
10 files changed, 51 insertions(+), 13 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 04166db..ae44b08 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1191,7 +1191,7 @@ public class HiveConf extends Configuration {
*/
@Deprecated
METASTORE_EVENT_MESSAGE_FACTORY("hive.metastore.event.message.factory",
- "org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder",
+ "org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder",
"Factory class for making encoding and decoding messages in the events generated."),
/**
* @deprecated Use MetastoreConf.EXECUTE_SET_UGI
diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
index da08d2f..5b43323 100644
--- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
+++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
@@ -80,7 +82,6 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertArrayEquals;
-import org.apache.hadoop.util.Shell;
import javax.annotation.Nullable;
@@ -120,6 +121,8 @@ public class TestHCatClient {
System.setProperty(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS.varname,
DbNotificationListener.class.getName()); // turn on db notification listener on metastore
+ System.setProperty(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+ JSONMessageEncoder.class.getName());
msPort = MetaStoreTestUtils.startMetaStoreWithRetry();
securityManager = System.getSecurityManager();
System.setSecurityManager(new NoExitSecurityManager());
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java
index c4c8f50..ae7e572 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
@@ -66,8 +67,9 @@ public class TestHCatClientNotification {
@BeforeClass
public static void setupClient() throws Exception {
- HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS,
- DbNotificationListener.class.getName());
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, DbNotificationListener.class.getName());
+ conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY, JSONMessageEncoder.class.getName());
hCatClient = HCatClient.create(conf);
md = MessageFactory.getInstance().getDeserializer();
}
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 9eacfd8..0c3e0466 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -274,6 +274,7 @@ public class TestDbNotificationListener {
conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, DummyRawStoreFailEvent.class.getName());
MetastoreConf.setTimeVar(conf, MetastoreConf.ConfVars.EVENT_DB_LISTENER_CLEAN_INTERVAL, CLEANUP_SLEEP_TIME, TimeUnit.SECONDS);
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY, JSONMessageEncoder.class.getName());
conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
SessionState.start(new CliSessionState(conf));
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java
index c16799d..747b7a8 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcidTablesWithJsonMessage.java
@@ -18,12 +18,16 @@
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TestRule;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
public class TestReplAcidTablesWithJsonMessage extends TestReplicationScenariosAcidTables {
@@ -32,7 +36,10 @@ public class TestReplAcidTablesWithJsonMessage extends TestReplicationScenariosA
@BeforeClass
public static void classLevelSetup() throws Exception {
- internalBeforeClassSetup(Collections.emptyMap(), TestReplAcidTablesWithJsonMessage.class);
+ Map<String, String> overrides = new HashMap<>();
+ overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+ JSONMessageEncoder.class.getCanonicalName());
+ internalBeforeClassSetup(overrides, TestReplAcidTablesWithJsonMessage.class);
}
@Before
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java
index 0ec0275..05517c7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplAcrossInstancesWithJsonMessageFormat.java
@@ -18,13 +18,16 @@
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TestRule;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
public class TestReplAcrossInstancesWithJsonMessageFormat
extends TestReplicationScenariosAcrossInstances {
@@ -34,7 +37,10 @@ public class TestReplAcrossInstancesWithJsonMessageFormat
@BeforeClass
public static void classLevelSetup() throws Exception {
- internalBeforeClassSetup(Collections.emptyMap(), TestReplicationScenarios.class);
+ Map<String, String> overrides = new HashMap<>();
+ overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+ JSONMessageEncoder.class.getCanonicalName());
+ internalBeforeClassSetup(overrides, TestReplicationScenarios.class);
}
@Before
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java
index 422508d..cd273f0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplIncrementalLoadAcidTablesWithJsonMessage.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -25,6 +27,8 @@ import org.junit.rules.TestRule;
import org.junit.Ignore;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
@Ignore
public class TestReplIncrementalLoadAcidTablesWithJsonMessage
@@ -35,8 +39,10 @@ public class TestReplIncrementalLoadAcidTablesWithJsonMessage
@BeforeClass
public static void classLevelSetup() throws Exception {
- internalBeforeClassSetup(Collections.emptyMap(),
- TestReplIncrementalLoadAcidTablesWithJsonMessage.class);
+ Map<String, String> overrides = new HashMap<>();
+ overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+ JSONMessageEncoder.class.getCanonicalName());
+ internalBeforeClassSetup(overrides, TestReplIncrementalLoadAcidTablesWithJsonMessage.class);
}
@Before
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java
index 0151ed0..5e7bf7e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplTableMigrationWithJsonFormat.java
@@ -17,13 +17,19 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.junit.BeforeClass;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
public class TestReplTableMigrationWithJsonFormat extends TestReplicationWithTableMigration {
@BeforeClass
public static void classLevelSetup() throws Exception {
- internalBeforeClassSetup(Collections.emptyMap());
+ Map<String, String> overrides = new HashMap<>();
+ overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+ JSONMessageEncoder.class.getCanonicalName());
+ internalBeforeClassSetup(overrides);
}
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java
index f76dc1d..6d9c31f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplWithJsonMessageFormat.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -24,6 +26,8 @@ import org.junit.rules.TestRule;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
public class TestReplWithJsonMessageFormat extends TestReplicationScenarios {
@Rule
@@ -33,7 +37,10 @@ public class TestReplWithJsonMessageFormat extends TestReplicationScenarios {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- internalBeforeClassSetup(Collections.emptyMap(), false);
+ Map<String, String> overrides = new HashMap<>();
+ overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+ JSONMessageEncoder.class.getCanonicalName());
+ internalBeforeClassSetup(overrides, false);
}
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 3a9912f..ce352e5 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -544,7 +544,7 @@ public class MetastoreConf {
"Alternatively, configure hive.metastore.transactional.event.listeners to ensure both are invoked in same JDO transaction."),
EVENT_MESSAGE_FACTORY("metastore.event.message.factory",
"hive.metastore.event.message.factory",
- "org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder",
+ "org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder",
"Factory class for making encoding and decoding messages in the events generated."),
EVENT_NOTIFICATION_PARAMETERS_EXCLUDE_PATTERNS("metastore.notification.parameters.exclude.patterns",
"hive.metastore.notification.parameters.exclude.patterns", "",