You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gs...@apache.org on 2023/09/07 18:35:28 UTC
[hive] branch master updated: HIVE-27113: Increasing default for hive.thrift.client.max.message.size to 2GB (#4621)(Riju Trivedi, Reviewed by Attila Turoczy, Sai Hemanth Gantasala)
This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9ac245fc5cd HIVE-27113: Increasing default for hive.thrift.client.max.message.size to 2GB (#4621)(Riju Trivedi, Reviewed by Attila Turoczy, Sai Hemanth Gantasala)
9ac245fc5cd is described below
commit 9ac245fc5cd96355f5f437663363ea926473cdfb
Author: rtrivedi12 <32...@users.noreply.github.com>
AuthorDate: Fri Sep 8 00:05:18 2023 +0530
HIVE-27113: Increasing default for hive.thrift.client.max.message.size to 2GB (#4621)(Riju Trivedi, Reviewed by Attila Turoczy, Sai Hemanth Gantasala)
---
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 8 ++++----
.../org/apache/hive/minikdc/TestRemoteHiveMetaStoreKerberos.java | 2 +-
.../src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java | 2 +-
.../hive/service/cli/thrift/RetryingThriftCLIServiceClient.java | 4 ++--
.../org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java | 3 ++-
.../java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 7 ++++---
6 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4b0c801b491..afaeed3fdf4 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2928,10 +2928,10 @@ public class HiveConf extends Configuration {
HIVE_STATS_MAX_NUM_STATS("hive.stats.max.num.stats", (long) 10000,
"When the number of stats to be updated is huge, this value is used to control the number of \n" +
" stats to be sent to HMS for update."),
- HIVE_THRIFT_CLIENT_MAX_MESSAGE_SIZE("hive.thrift.client.max.message.size", "1gb",
- new SizeValidator(-1L, true, (long) Integer.MAX_VALUE, true),
- "Thrift client configuration for max message size. 0 or -1 will use the default defined in the Thrift " +
- "library. The upper limit is 2147483648 bytes (or 2gb)."),
+ HIVE_THRIFT_CLIENT_MAX_MESSAGE_SIZE("hive.thrift.client.max.message.size", (2*1024*1024*1024L)-1L,
+ new SizeValidator(-1L, true, (long) Integer.MAX_VALUE, true),
+ "Thrift client configuration for max message size. 0 or -1 will use the default defined in the Thrift " +
+ "library. The upper limit is 2147483647 bytes."),
// Concurrency
HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false,
"Whether Hive supports concurrency control or not. \n" +
diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestRemoteHiveMetaStoreKerberos.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestRemoteHiveMetaStoreKerberos.java
index c24cb5b62bb..4bf52e53b4c 100644
--- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestRemoteHiveMetaStoreKerberos.java
+++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestRemoteHiveMetaStoreKerberos.java
@@ -77,7 +77,7 @@ public class TestRemoteHiveMetaStoreKerberos extends TestRemoteHiveMetaStore {
MetastoreConf.setVar(clientConf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port);
// set to a low value to prove THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE is being honored
// (it should throw an exception)
- MetastoreConf.setVar(clientConf, ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE, "1024");
+ MetastoreConf.setLongVar(clientConf, ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE, 1024L);
HiveMetaStoreClient limitedClient = new HiveMetaStoreClient(clientConf);
Exception expectedException = assertThrows(TTransportException.class, () -> {
limitedClient.listPartitions(dbName, tblName, (short)-1);
diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java
index 7ca74efb648..8b63b339086 100644
--- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java
+++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java
@@ -98,7 +98,7 @@ public class TestSSLWithMiniKdc {
MetastoreConf.setVar(clientConf, MetastoreConf.ConfVars.THRIFT_URIS, "thrift://localhost:" + miniHS2.getHmsPort());
// set to a low value to prove THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE is being honored
// (it should throw an exception)
- MetastoreConf.setVar(clientConf, MetastoreConf.ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE, "512");
+ MetastoreConf.setLongVar(clientConf, MetastoreConf.ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE, 512L);
HiveMetaStoreClient limitedClient = new HiveMetaStoreClient(clientConf);
String dbName = "default";
String tableName = "testThriftMaxMessageSize";
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java b/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
index 9079c652d0a..37068e99e0e 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
@@ -310,8 +310,8 @@ public class RetryingThriftCLIServiceClient implements InvocationHandler {
String host = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST);
int port = conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT);
- int maxThriftMessageSize = (int) conf.getSizeVar(HiveConf.ConfVars.HIVE_THRIFT_CLIENT_MAX_MESSAGE_SIZE);
- LOG.info("Connecting to " + host + ":" + port);
+ int maxThriftMessageSize = (int) Math.min(conf.getLongVar(HiveConf.ConfVars.HIVE_THRIFT_CLIENT_MAX_MESSAGE_SIZE),Integer.MAX_VALUE);
+ LOG.info("Connecting to " + host + ":" + port+ " using a thrift max message of size: "+ maxThriftMessageSize);
transport = HiveAuthUtils.getSocketTransport(host, port, 0, maxThriftMessageSize);
((TSocket) transport).setTimeout((int) conf.getTimeVar(HiveConf.ConfVars.SERVER_READ_SOCKET_TIMEOUT,
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 503183b4532..646b978c0bc 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -610,7 +610,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
private <T extends TTransport> T configureThriftMaxMessageSize(T transport) {
- int maxThriftMessageSize = (int) MetastoreConf.getSizeVar(conf, ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE);
+ int maxThriftMessageSize = Math.min(MetastoreConf.getIntVar(
+ conf, ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE),Integer.MAX_VALUE);
if (maxThriftMessageSize > 0) {
if (transport.getConfiguration() == null) {
LOG.warn("TTransport {} is returning a null Configuration, Thrift max message size is not getting configured",
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 6f597a1739d..691d93edeb0 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1477,9 +1477,10 @@ public class MetastoreConf {
" corresponding service discovery servers e.g. a zookeeper. Otherwise they are " +
"used as URIs for remote metastore."),
THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE("metastore.thrift.client.max.message.size",
- "hive.thrift.client.max.message.size", "1gb", new SizeValidator(-1L, true, (long) Integer.MAX_VALUE, true),
- "Thrift client configuration for max message size. 0 or -1 will use the default defined in the Thrift " +
- "library. The upper limit is 2147483648 bytes (or 2gb)."),
+ "hive.thrift.client.max.message.size", (2*1024*1024*1024L)-1L,
+ new SizeValidator(-1L, true, (long) Integer.MAX_VALUE, true),
+ "Thrift client configuration for max message size. 0 or -1 will use the default defined in the Thrift " +
+ "library. The upper limit is 2147483647 bytes"),
THRIFT_SERVICE_DISCOVERY_MODE("metastore.service.discovery.mode",
"hive.metastore.service.discovery.mode",
"",