You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2017/11/28 19:44:30 UTC
hive git commit: HIVE-18147 : Tests can fail with
java.net.BindException: Address already in use (Janaki Lahorani,
reviewed by Andrew Sherman and Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master 525cfa7be -> 542b165c3
HIVE-18147 : Tests can fail with java.net.BindException: Address already in use (Janaki Lahorani, reviewed by Andrew Sherman and Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/542b165c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/542b165c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/542b165c
Branch: refs/heads/master
Commit: 542b165c3f042f9d726d2ed831adfc5a89ce369f
Parents: 525cfa7
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Tue Nov 28 11:11:39 2017 -0800
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Tue Nov 28 11:30:57 2017 -0800
----------------------------------------------------------------------
.../apache/hive/hcatalog/cli/TestPermsGrp.java | 4 +-
.../mapreduce/TestHCatPartitionPublish.java | 3 +-
.../hive/hcatalog/templeton/TestWebHCatE2e.java | 43 ++++++++++++++------
.../AbstractTestAuthorizationApiAuthorizer.java | 3 +-
.../hadoop/hive/metastore/TestFilterHooks.java | 3 +-
...TestHiveMetaStoreWithEnvironmentContext.java | 3 +-
.../TestMetaStoreEndFunctionListener.java | 3 +-
.../metastore/TestMetaStoreEventListener.java | 3 +-
.../TestMetaStoreEventListenerOnlyOnCommit.java | 3 +-
.../metastore/TestMetaStoreInitListener.java | 3 +-
.../hive/metastore/TestMetaStoreMetrics.java | 7 ++--
.../hive/metastore/TestRemoteHiveMetaStore.java | 3 +-
.../TestRemoteHiveMetaStoreIpAddress.java | 5 +--
.../hive/metastore/TestRetryingHMSHandler.java | 3 +-
...estDDLWithRemoteMetastoreSecondNamenode.java | 3 +-
.../security/StorageBasedMetastoreTestBase.java | 5 +--
.../TestAuthorizationPreEventListener.java | 4 +-
.../TestClientSideAuthorizationProvider.java | 4 +-
.../TestMetastoreAuthorizationProvider.java | 5 +--
.../TestMultiAuthorizationPreEventListener.java | 6 +--
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 37 ++++++++++++++---
.../hive/metastore/MetaStoreTestUtils.java | 32 +++++++++++++++
.../thrift/ThriftCliServiceTestWithCookie.java | 26 ++++++++----
23 files changed, 134 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
index 80c5d63..e5aa2b8 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
@@ -80,9 +80,7 @@ public class TestPermsGrp extends TestCase {
return;
}
-
- msPort = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge());
+ msPort = MetaStoreTestUtils.startMetaStoreWithRetry();
isServerRunning = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
index a000642..6711a3e 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
@@ -105,9 +105,8 @@ public class TestHCatPartitionPublish {
return;
}
- msPort = MetaStoreTestUtils.findFreePort();
+ msPort = MetaStoreTestUtils.startMetaStoreWithRetry();
- MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge());
Thread.sleep(10000);
isServerRunning = true;
securityManager = System.getSecurityManager();
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
index 341314b..55cd5b1 100644
--- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
+++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
@@ -70,21 +70,38 @@ public class TestWebHCatE2e {
private static final String charSet = "UTF-8";
@BeforeClass
- public static void startHebHcatInMem() {
- int webhcatPort = 50111;
- try {
- //in case concurrent tests are running on the same machine
- webhcatPort = MetaStoreTestUtils.findFreePort();
+ public static void startHebHcatInMem() throws Exception {
+ Exception webhcatException = null;
+ int webhcatPort = 0;
+ boolean webhcatStarted = false;
+
+ for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) {
+ try {
+ if (tryCount == MetaStoreTestUtils.RETRY_COUNT - 1) {
+ /* Last try to get a port. Just use default 50111. */
+ webhcatPort = 50111;
+ LOG.warn("Unable to find free port; using default: " + webhcatPort);
+ }
+ else {
+ webhcatPort = MetaStoreTestUtils.findFreePort();
+ }
+ templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort));
+ templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true",
+ "-D" + AppConfig.PORT + "=" + webhcatPort });
+ LOG.info("Starting Main; WebHCat using port: " + webhcatPort);
+ templetonServer.run();
+ LOG.info("Main started");
+ webhcatStarted = true;
+ break;
+ } catch (Exception ce) {
+ LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed");
+ webhcatException = ce;
+ }
}
- catch (IOException ex) {
- LOG.warn("Unable to find free port; using default: " + webhcatPort);
+
+ if (!webhcatStarted) {
+ throw webhcatException;
}
- templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort));
- templetonServer = new Main(new String[] {"-D" +
- AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort});
- LOG.info("Starting Main; WebHCat using port: " + webhcatPort);
- templetonServer.run();
- LOG.info("Main started");
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
index ebdfab2..bd8332c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
@@ -57,8 +57,7 @@ public abstract class AbstractTestAuthorizationApiAuthorizer {
hiveConf = new HiveConf();
if (isRemoteMetastoreMode) {
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
}
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
index 6a2bbd2..8366fab 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
@@ -174,9 +174,8 @@ public class TestFilterHooks {
hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName());
UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName());
- int port = MetaStoreTestUtils.findFreePort();
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf);
SessionState.start(new CliSessionState(hiveConf));
msc = new HiveMetaStoreClient(hiveConf);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
index 3fa4b1e..5267dca 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
@@ -73,8 +73,7 @@ public class TestHiveMetaStoreWithEnvironmentContext extends TestCase {
System.setProperty("hive.metastore.event.listeners",
DummyListener.class.getName());
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
index 27bc791..3bb340e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
@@ -48,8 +48,7 @@ public class TestMetaStoreEndFunctionListener extends TestCase {
DummyPreListener.class.getName());
System.setProperty("hive.metastore.end.function.listeners",
DummyEndFunctionListener.class.getName());
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
index 4df2f86..7219774 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
@@ -94,11 +94,10 @@ public class TestMetaStoreEventListener extends TestCase {
System.setProperty("hive.metastore.pre.event.listeners",
DummyPreListener.class.getName());
- int port = MetaStoreTestUtils.findFreePort();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, metaConfVal);
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf);
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
index d9da422..a905324 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
@@ -50,8 +50,7 @@ public class TestMetaStoreEventListenerOnlyOnCommit extends TestCase {
System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname,
DummyRawStoreControlledCommit.class.getName());
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
index 9578ef9..3d58819 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
@@ -41,8 +41,7 @@ public class TestMetaStoreInitListener extends TestCase {
super.setUp();
System.setProperty("hive.metastore.init.hooks",
DummyMetaStoreInitListener.class.getName());
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index a899739..9d241a8 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -40,10 +40,7 @@ public class TestMetaStoreMetrics {
@BeforeClass
public static void before() throws Exception {
- int port = MetaStoreTestUtils.findFreePort();
-
hiveConf = new HiveConf(TestMetaStoreMetrics.class);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
@@ -52,7 +49,9 @@ public class TestMetaStoreMetrics {
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
//Increments one HMS connection
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf);
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf);
+
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
//Increments one HMS connection (Hive.get())
SessionState.start(new CliSessionState(hiveConf));
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
index 19bed9d..d4a809f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
@@ -42,9 +42,8 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
return;
}
- port = MetaStoreTestUtils.findFreePort();
+ port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf);
System.out.println("Starting MetaStore Server on port " + port);
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf);
isServerStarted = true;
// This is default case with setugi off for both client and server
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
index 449fe71..9ba1d75 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
@@ -48,11 +48,10 @@ public class TestRemoteHiveMetaStoreIpAddress extends TestCase {
return;
}
- int port = MetaStoreTestUtils.findFreePort();
- System.out.println("Starting MetaStore Server on port " + port);
System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname,
IpAddressListener.class.getName());
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
+ System.out.println("Started MetaStore Server on port " + port);
isServerStarted = true;
// This is default case with setugi off for both client and server
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
index 41494d1..2b7c5a1 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
@@ -53,8 +53,7 @@ public class TestRetryingHMSHandler extends TestCase {
super.setUp();
System.setProperty("hive.metastore.pre.event.listeners",
AlternateFailurePreListener.class.getName());
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
index 4ce2731..179eed9 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
@@ -80,8 +80,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase {
SessionState.start(conf);
// Test with remote metastore service
- int port = MetaStoreTestUtils.findFreePort();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
index fa5ae80..dc3af3c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
@@ -62,9 +62,6 @@ public class StorageBasedMetastoreTestBase {
@Before
public void setUp() throws Exception {
-
- int port = MetaStoreTestUtils.findFreePort();
-
// Turn on metastore-side authorization
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
@@ -74,7 +71,7 @@ public class StorageBasedMetastoreTestBase {
InjectableDummyAuthenticator.class.getName());
clientHiveConf = createHiveConf();
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), clientHiveConf);
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry(clientHiveConf);
// Turn off client-side authorization
clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false);
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
index 102af6a..6a668aa 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
@@ -53,8 +53,6 @@ public class TestAuthorizationPreEventListener extends TestCase {
super.setUp();
- int port = MetaStoreTestUtils.findFreePort();
-
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname,
@@ -62,7 +60,7 @@ public class TestAuthorizationPreEventListener extends TestCase {
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
HadoopDefaultMetastoreAuthenticator.class.getName());
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
clientHiveConf = new HiveConf(this.getClass());
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
index 60273e7..57ff8c9 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
@@ -58,13 +58,11 @@ public class TestClientSideAuthorizationProvider extends TestCase {
super.setUp();
- int port = MetaStoreTestUtils.findFreePort();
-
// Turn off metastore-side authorization
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
"");
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
clientHiveConf = new HiveConf(this.getClass());
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
index f0196c6..871cf3d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
@@ -84,8 +84,6 @@ public class TestMetastoreAuthorizationProvider extends TestCase {
super.setUp();
- int port = MetaStoreTestUtils.findFreePort();
-
// Turn on metastore-side authorization
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
@@ -98,8 +96,7 @@ public class TestMetastoreAuthorizationProvider extends TestCase {
InjectableDummyAuthenticator.class.getName());
System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, "");
-
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
clientHiveConf = createHiveConf();
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
index 62835b0..2059370 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
@@ -47,10 +47,6 @@ public class TestMultiAuthorizationPreEventListener {
@BeforeClass
public static void setUp() throws Exception {
-
-
- int port = MetaStoreTestUtils.findFreePort();
-
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
@@ -62,7 +58,7 @@ public class TestMultiAuthorizationPreEventListener {
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
HadoopDefaultMetastoreAuthenticator.class.getName());
- MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge());
+ int port = MetaStoreTestUtils.startMetaStoreWithRetry();
clientHiveConf = new HiveConf();
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index e554cc8..ae78c19 100644
--- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapItUtils;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
-import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
@@ -75,6 +74,7 @@ public class MiniHS2 extends AbstractHiveService {
private final boolean cleanupLocalDirOnStartup;
private final boolean isMetastoreSecure;
private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
+ private boolean usePortsFromConf = false;
public enum MiniClusterType {
MR,
@@ -227,6 +227,7 @@ public class MiniHS2 extends AbstractHiveService {
this.isMetastoreRemote = isMetastoreRemote;
this.isMetastoreSecure = isMetastoreSecure;
this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup;
+ this.usePortsFromConf = usePortsFromConf;
baseDir = getBaseDir();
localFS = FileSystem.getLocal(hiveConf);
FileSystem fs;
@@ -334,18 +335,42 @@ public class MiniHS2 extends AbstractHiveService {
public void start(Map<String, String> confOverlay) throws Exception {
if (isMetastoreRemote) {
- int metaStorePort = MetaStoreTestUtils.findFreePort();
+ int metaStorePort = MetaStoreTestUtils.startMetaStoreWithRetry(getHiveConf());
getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
- MetaStoreTestUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), getHiveConf());
}
- hiveServer2 = new HiveServer2();
// Set confOverlay parameters
for (Map.Entry<String, String> entry : confOverlay.entrySet()) {
setConfProperty(entry.getKey(), entry.getValue());
}
- hiveServer2.init(getHiveConf());
- hiveServer2.start();
+
+ Exception hs2Exception = null;
+ boolean hs2Started = false;
+ for (int tryCount = 0; (tryCount < MetaStoreTestUtils.RETRY_COUNT); tryCount++) {
+ try {
+ hiveServer2 = new HiveServer2();
+ hiveServer2.init(getHiveConf());
+ hiveServer2.start();
+ hs2Started = true;
+ break;
+ } catch (Exception t) {
+ hs2Exception = t;
+ if (usePortsFromConf) {
+ hs2Started = false;
+ break;
+ } else {
+ HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT,
+ MetaStoreTestUtils.findFreePort());
+ HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT,
+ MetaStoreTestUtils.findFreePort());
+ }
+ }
+ }
+
+ if (!hs2Started) {
+ throw(hs2Exception);
+ }
+
waitForStartup();
setStarted(true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
index 28353c3..7df7ac5 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
import java.io.IOException;
+import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -35,6 +36,7 @@ import org.slf4j.LoggerFactory;
public class MetaStoreTestUtils {
private static final Logger LOG = LoggerFactory.getLogger("hive.log");
+ public static final int RETRY_COUNT = 10;
public static int startMetaStore() throws Exception {
return MetaStoreTestUtils.startMetaStore(HadoopThriftAuthBridge.getBridge(), null);
@@ -76,6 +78,36 @@ public class MetaStoreTestUtils {
MetaStoreTestUtils.loopUntilHMSReady(port);
}
+ public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception {
+ return MetaStoreTestUtils.startMetaStoreWithRetry(bridge, null);
+ }
+
+ public static int startMetaStoreWithRetry(HiveConf conf) throws Exception {
+ return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf);
+ }
+
+ public static int startMetaStoreWithRetry() throws Exception {
+ return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), null);
+ }
+
+ public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf)
+ throws Exception {
+ Exception metaStoreException = null;
+ int metaStorePort = 0;
+
+ for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) {
+ try {
+ metaStorePort = MetaStoreTestUtils.findFreePort();
+ MetaStoreTestUtils.startMetaStore(metaStorePort, bridge, conf);
+ return metaStorePort;
+ } catch (ConnectException ce) {
+ metaStoreException = ce;
+ }
+ }
+
+ throw metaStoreException;
+ }
+
/**
* A simple connect test to make sure that the metastore is up
* @throws Exception
http://git-wip-us.apache.org/repos/asf/hive/blob/542b165c/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
index 6660097..1dac783 100644
--- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
+++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
@@ -96,15 +96,27 @@ public class ThriftCliServiceTestWithCookie {
}
protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception {
- hiveServer2.init(hiveConf);
- // Start HiveServer2 with given config
- // Fail if server doesn't start
- try {
- hiveServer2.start();
- } catch (Throwable t) {
- t.printStackTrace();
+ Exception HS2Exception = null;
+ boolean HS2Started = false;
+ for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) {
+ try {
+ hiveServer2.init(hiveConf);
+ hiveServer2.start();
+ HS2Started = true;
+ break;
+ } catch (Exception t) {
+ HS2Exception = t;
+ port = MetaStoreTestUtils.findFreePort();
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port);
+ hiveServer2 = new HiveServer2();
+ }
+ }
+
+ if (!HS2Started) {
+ HS2Exception.printStackTrace();
fail();
}
+
// Wait for startup to complete
Thread.sleep(2000);
System.out.println("HiveServer2 started on port " + port);