You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2021/03/18 07:03:28 UTC
[iceberg] branch master updated: Hive: Avoid reset hive
configuration to default value. (#2075)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 62f8210 Hive: Avoid reset hive configuration to default value. (#2075)
62f8210 is described below
commit 62f8210b43ba91ea4a26d3b9ce0525ccf8ecfa7e
Author: dixingxing <di...@yeah.net>
AuthorDate: Thu Mar 18 15:03:16 2021 +0800
Hive: Avoid reset hive configuration to default value. (#2075)
* Add test for iceberg-[2070]
* Fix test and codestyle
* Add TestHiveClientPool
* Delete unnecessary test case
* Delete unnecessary test case
* Delete misunderstanding test code.
Co-authored-by: dixingxing <di...@autohome.com.cn>
---
.../java/org/apache/iceberg/hive/ClientPool.java | 6 ++
.../org/apache/iceberg/hive/HiveClientPool.java | 7 +++
.../apache/iceberg/hive/TestHiveClientPool.java | 70 ++++++++++++++++++++++
3 files changed, 83 insertions(+)
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/ClientPool.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/ClientPool.java
index 7fe9b66..cccd0da 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/ClientPool.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/ClientPool.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.hive;
import java.io.Closeable;
import java.util.ArrayDeque;
import java.util.Deque;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,6 +111,11 @@ public abstract class ClientPool<C, E extends Exception> implements Closeable {
}
}
+ @VisibleForTesting
+ int poolSize() {
+ return poolSize;
+ }
+
private C get() throws InterruptedException {
Preconditions.checkState(!closed, "Cannot get a client from a closed pool");
while (true) {
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveClientPool.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
index 3a317b2..3b17334 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.iceberg.common.DynConstructors;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransportException;
@@ -41,6 +42,7 @@ public class HiveClientPool extends ClientPool<HiveMetaStoreClient, TException>
public HiveClientPool(int poolSize, Configuration conf) {
super(poolSize, TTransportException.class);
this.hiveConf = new HiveConf(conf, HiveClientPool.class);
+ this.hiveConf.addResource(conf);
}
@Override
@@ -89,4 +91,9 @@ public class HiveClientPool extends ClientPool<HiveMetaStoreClient, TException>
protected void close(HiveMetaStoreClient client) {
client.close();
}
+
+ @VisibleForTesting
+ HiveConf hiveConf() {
+ return hiveConf;
+ }
}
diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java
new file mode 100644
index 0000000..f5e2ea6
--- /dev/null
+++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hive;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHiveClientPool {
+
+ private static final String HIVE_SITE_CONTENT = "<?xml version=\"1.0\"?>\n" +
+ "<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n" +
+ "<configuration>\n" +
+ " <property>\n" +
+ " <name>hive.metastore.sasl.enabled</name>\n" +
+ " <value>true</value>\n" +
+ " </property>\n" +
+ "</configuration>\n";
+
+ @Test
+ public void testConf() {
+ HiveConf conf = createHiveConf();
+ conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, "file:/mywarehouse/");
+ conf.setInt("iceberg.hive.client-pool-size", 10);
+
+ HiveClientPool clientPool = new HiveClientPool(conf);
+ HiveConf clientConf = clientPool.hiveConf();
+
+ Assert.assertEquals(conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
+ clientConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname));
+ Assert.assertEquals(conf.get("iceberg.hive.client-pool-size"), clientConf.get("iceberg.hive.client-pool-size"));
+ Assert.assertEquals(10, clientPool.poolSize());
+
+ // 'hive.metastore.sasl.enabled' should be 'true' as defined in xml
+ Assert.assertEquals(conf.get(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname),
+ clientConf.get(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname));
+ Assert.assertTrue(clientConf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL));
+ }
+
+ private HiveConf createHiveConf() {
+ HiveConf hiveConf = new HiveConf();
+ try (InputStream inputStream = new ByteArrayInputStream(HIVE_SITE_CONTENT.getBytes(StandardCharsets.UTF_8))) {
+ hiveConf.addResource(inputStream, "for_test");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return hiveConf;
+ }
+}