[iceberg] branch master updated: Hive: Make the TestHiveMetastore connection pool size configurable (#1629)

[op...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

openinx pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 50d10f3  Hive: Make the TestHiveMetastore connection pool size configurable (#1629)
50d10f3 is described below

commit 50d10f39fb9836e8bb38535637860dbad320ecc1
Author: pvary <pv...@cloudera.com>
AuthorDate: Mon Oct 19 12:07:23 2020 +0200

    Hive: Make the TestHiveMetastore connection pool size configurable (#1629)
---
 .../org/apache/iceberg/hive/TestHiveMetastore.java   | 20 ++++++++++++++++----
 .../mr/hive/HiveIcebergStorageHandlerBaseTest.java   |  5 ++++-
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
index 04007ad..d2aab50 100644
--- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
+++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
@@ -54,6 +54,7 @@ import static java.nio.file.attribute.PosixFilePermissions.fromString;
 public class TestHiveMetastore {
 
   private static final String DEFAULT_DATABASE_NAME = "default";
+  private static final int DEFAULT_POOL_SIZE = 5;
 
   // create the metastore handlers based on whether we're working with Hive2 or Hive3 dependencies
   // we need to do this because there is a breaking API change between Hive2 and Hive3
@@ -74,7 +75,18 @@ public class TestHiveMetastore {
   private HiveMetaStore.HMSHandler baseHandler;
   private HiveClientPool clientPool;
 
+  /**
+   * Starts a TestHiveMetastore with the default connection pool size (5).
+   */
   public void start() {
+    start(DEFAULT_POOL_SIZE);
+  }
+
+  /**
+   * Starts a TestHiveMetastore with a provided connection pool size.
+   * @param poolSize The number of threads in the executor pool
+   */
+  public void start(int poolSize) {
     try {
       this.hiveLocalDir = createTempDirectory("hive", asFileAttribute(fromString("rwxrwxrwx"))).toFile();
       File derbyLogFile = new File(hiveLocalDir, "derby.log");
@@ -84,7 +96,7 @@ public class TestHiveMetastore {
       TServerSocket socket = new TServerSocket(0);
       int port = socket.getServerSocket().getLocalPort();
       this.hiveConf = newHiveConf(port);
-      this.server = newThriftServer(socket, hiveConf);
+      this.server = newThriftServer(socket, poolSize, hiveConf);
       this.executorService = Executors.newSingleThreadExecutor();
       this.executorService.submit(() -> server.serve());
 
@@ -156,7 +168,7 @@ public class TestHiveMetastore {
     }
   }
 
-  private TServer newThriftServer(TServerSocket socket, HiveConf conf) throws Exception {
+  private TServer newThriftServer(TServerSocket socket, int poolSize, HiveConf conf) throws Exception {
     HiveConf serverConf = new HiveConf(conf);
     serverConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:" + getDerbyPath() + ";create=true");
     baseHandler = HMS_HANDLER_CTOR.newInstance("new db based metaserver", serverConf);
@@ -166,8 +178,8 @@ public class TestHiveMetastore {
         .processor(new TSetIpAddressProcessor<>(handler))
         .transportFactory(new TTransportFactory())
         .protocolFactory(new TBinaryProtocol.Factory())
-        .minWorkerThreads(5)
-        .maxWorkerThreads(15);
+        .minWorkerThreads(poolSize)
+        .maxWorkerThreads(poolSize);
 
     return new TThreadPoolServer(args);
   }
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
index d3cea88..63e32dd 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
@@ -108,6 +108,8 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
       ImmutableSet.of("bucketing_version", StatsSetupConst.ROW_COUNT,
           StatsSetupConst.RAW_DATA_SIZE, StatsSetupConst.TOTAL_SIZE, StatsSetupConst.NUM_FILES);
 
+  private static final int METASTORE_POOL_SIZE = 15;
+
   // before variables
   protected static TestHiveMetastore metastore;
 
@@ -119,7 +121,8 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
   @BeforeClass
   public static void beforeClass() {
     metastore = new TestHiveMetastore();
-    metastore.start();
+    // We need to use increased pool size in these tests. See: #1620
+    metastore.start(METASTORE_POOL_SIZE);
   }
 
   @AfterClass