You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2017/12/03 10:14:08 UTC
hive git commit: HIVE-18147 : Tests can fail with
java.net.BindException: Address already in use (Janaki Lahorani,
reviewed by Andrew Sherman and Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/branch-2 059844e48 -> 1f7e3be05
HIVE-18147 : Tests can fail with java.net.BindException: Address already in use (Janaki Lahorani, reviewed by Andrew Sherman and Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f7e3be0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f7e3be0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f7e3be0
Branch: refs/heads/branch-2
Commit: 1f7e3be05982a4cb41f9316bf2c012767b5b2a5f
Parents: 059844e
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Sun Dec 3 15:36:15 2017 +0530
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Sun Dec 3 15:43:15 2017 +0530
----------------------------------------------------------------------
.../apache/hive/hcatalog/cli/TestPermsGrp.java | 4 +-
.../mapreduce/TestHCatPartitionPublish.java | 4 +-
.../hive/hcatalog/templeton/TestWebHCatE2e.java | 41 +++++++++++++-------
.../AbstractTestAuthorizationApiAuthorizer.java | 3 +-
.../hadoop/hive/metastore/TestFilterHooks.java | 3 +-
.../metastore/TestHiveMetaStoreStatsMerge.java | 3 +-
...TestHiveMetaStoreWithEnvironmentContext.java | 3 +-
.../TestMetaStoreEndFunctionListener.java | 3 +-
.../metastore/TestMetaStoreEventListener.java | 3 +-
.../TestMetaStoreEventListenerOnlyOnCommit.java | 3 +-
.../metastore/TestMetaStoreInitListener.java | 3 +-
.../hive/metastore/TestMetaStoreMetrics.java | 7 ++--
.../hive/metastore/TestRemoteHiveMetaStore.java | 3 +-
.../TestRemoteHiveMetaStoreIpAddress.java | 5 +--
.../hive/metastore/TestRetryingHMSHandler.java | 3 +-
...estDDLWithRemoteMetastoreSecondNamenode.java | 3 +-
.../hive/ql/security/FolderPermissionBase.java | 3 +-
.../security/StorageBasedMetastoreTestBase.java | 5 +--
.../TestAuthorizationPreEventListener.java | 4 +-
.../TestClientSideAuthorizationProvider.java | 4 +-
.../TestMetastoreAuthorizationProvider.java | 5 +--
.../TestMultiAuthorizationPreEventListener.java | 6 +--
.../thrift/ThriftCliServiceMessageSizeTest.java | 24 +++++++++---
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 37 +++++++++++++++---
.../hadoop/hive/metastore/MetaStoreUtils.java | 33 ++++++++++++++++
.../thrift/ThriftCliServiceTestWithCookie.java | 25 +++++++++---
26 files changed, 155 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
index 66a5dd4..2375f5d 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
@@ -79,9 +79,7 @@ public class TestPermsGrp extends TestCase {
return;
}
-
- msPort = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(msPort, ShimLoader.getHadoopThriftAuthBridge());
+ msPort = MetaStoreUtils.startMetaStoreWithRetry();
isServerRunning = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
index 69874bc..777fc9c 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
@@ -104,10 +104,8 @@ public class TestHCatPartitionPublish {
return;
}
- msPort = MetaStoreUtils.findFreePort();
+ msPort = MetaStoreUtils.startMetaStoreWithRetry();
- MetaStoreUtils.startMetaStore(msPort, ShimLoader
- .getHadoopThriftAuthBridge());
Thread.sleep(10000);
isServerRunning = true;
securityManager = System.getSecurityManager();
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
index b62716e..841e2ff 100644
--- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
+++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java
@@ -70,21 +70,36 @@ public class TestWebHCatE2e {
private static final String charSet = "UTF-8";
@BeforeClass
- public static void startHebHcatInMem() {
- int webhcatPort = 50111;
- try {
- //in case concurrent tests are running on the same machine
- webhcatPort = MetaStoreUtils.findFreePort();
+ public static void startHebHcatInMem() throws Exception {
+ Exception webhcatException = null;
+ int webhcatPort = 0;
+ boolean webhcatStarted = false;
+
+ for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) {
+ try {
+ if (tryCount == MetaStoreUtils.RETRY_COUNT - 1) {
+ // down to the last try. try default port 50111
+ webhcatPort = 50111;
+ } else {
+ webhcatPort = MetaStoreUtils.findFreePort();
+ }
+ templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort));
+ templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true",
+ "-D" + AppConfig.PORT + "=" + webhcatPort });
+ LOG.info("Starting Main; WebHCat using port: " + webhcatPort);
+ templetonServer.run();
+ LOG.info("Main started");
+ webhcatStarted = true;
+ break;
+ } catch (Exception ce) {
+ LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed");
+ webhcatException = ce;
+ }
}
- catch (IOException ex) {
- LOG.warn("Unable to find free port; using default: " + webhcatPort);
+
+ if (!webhcatStarted) {
+ throw webhcatException;
}
- templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort));
- templetonServer = new Main(new String[] {"-D" +
- AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort});
- LOG.info("Starting Main; WebHCat using port: " + webhcatPort);
- templetonServer.run();
- LOG.info("Main started");
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
index 341c7ba..e72a585 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
@@ -57,8 +57,7 @@ public abstract class AbstractTestAuthorizationApiAuthorizer {
hiveConf = new HiveConf();
if (isRemoteMetastoreMode) {
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
}
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
index 1073abb..42859d4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
@@ -173,9 +173,8 @@ public class TestFilterHooks {
hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName());
UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName());
- int port = MetaStoreUtils.findFreePort();
+ int port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
SessionState.start(new CliSessionState(hiveConf));
msc = new HiveMetaStoreClient(hiveConf);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java
index 600aef4..4aa527c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java
@@ -76,8 +76,7 @@ public class TestHiveMetaStoreStatsMerge extends TestCase {
System.setProperty("hive.metastore.event.listeners",
DummyListener.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
index d6e4fb7..e999bfb 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java
@@ -73,8 +73,7 @@ public class TestHiveMetaStoreWithEnvironmentContext extends TestCase {
System.setProperty("hive.metastore.event.listeners",
DummyListener.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
index 1e78ff1..1158b76 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
@@ -47,8 +47,7 @@ public class TestMetaStoreEndFunctionListener extends TestCase {
DummyPreListener.class.getName());
System.setProperty("hive.metastore.end.function.listeners",
DummyEndFunctionListener.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
index fd4527e..2ad58cf 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
@@ -92,8 +92,7 @@ public class TestMetaStoreEventListener extends TestCase {
System.setProperty("hive.metastore.pre.event.listeners",
DummyPreListener.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
index 0c3e703..84ebbe1 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java
@@ -50,8 +50,7 @@ public class TestMetaStoreEventListenerOnlyOnCommit extends TestCase {
System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname,
DummyRawStoreControlledCommit.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
index e8171e5..b6426ff 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
@@ -42,8 +42,7 @@ public class TestMetaStoreInitListener extends TestCase {
super.setUp();
System.setProperty("hive.metastore.init.hooks",
DummyMetaStoreInitListener.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index b462b2a..8d6a45b 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -42,10 +42,7 @@ public class TestMetaStoreMetrics {
@BeforeClass
public static void before() throws Exception {
- int port = MetaStoreUtils.findFreePort();
-
hiveConf = new HiveConf(TestMetaStoreMetrics.class);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
@@ -58,7 +55,9 @@ public class TestMetaStoreMetrics {
metrics = (CodahaleMetrics) MetricsFactory.getInstance();
//Increments one HMS connection
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
+ int port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf);
+
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
//Increments one HMS connection (Hive.get())
SessionState.start(new CliSessionState(hiveConf));
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
index ef02968..225cf0a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
@@ -42,9 +42,8 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
return;
}
- port = MetaStoreUtils.findFreePort();
+ port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf);
System.out.println("Starting MetaStore Server on port " + port);
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
isServerStarted = true;
// This is default case with setugi off for both client and server
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
index 63eea27..1c1c3e6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
@@ -48,11 +48,10 @@ public class TestRemoteHiveMetaStoreIpAddress extends TestCase {
return;
}
- int port = MetaStoreUtils.findFreePort();
- System.out.println("Starting MetaStore Server on port " + port);
System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname,
IpAddressListener.class.getName());
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
+ System.out.println("Starting MetaStore Server on port " + port);
isServerStarted = true;
// This is default case with setugi off for both client and server
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
index 99c09b7..abe2416 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java
@@ -53,8 +53,7 @@ public class TestRetryingHMSHandler extends TestCase {
super.setUp();
System.setProperty("hive.metastore.pre.event.listeners",
AlternateFailurePreListener.class.getName());
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
index bfb25aa..a9bb4ad 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
@@ -79,8 +79,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase {
SessionState.start(conf);
// Test with remote metastore service
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
index bf6b7e1..63cd732 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
@@ -100,8 +100,7 @@ public abstract class FolderPermissionBase {
conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true);
conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ MetaStoreUtils.startMetaStoreWithRetry();
SessionState.start(new CliSessionState(conf));
driver = new Driver(conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
index 36f31f4..f6ac493 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
@@ -63,9 +63,6 @@ public class StorageBasedMetastoreTestBase {
@Before
public void setUp() throws Exception {
-
- int port = MetaStoreUtils.findFreePort();
-
// Turn on metastore-side authorization
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
@@ -75,7 +72,7 @@ public class StorageBasedMetastoreTestBase {
InjectableDummyAuthenticator.class.getName());
clientHiveConf = createHiveConf();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), clientHiveConf);
+ int port = MetaStoreUtils.startMetaStoreWithRetry(clientHiveConf);
// Turn off client-side authorization
clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
index b0da884..b357a55 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
@@ -53,8 +53,6 @@ public class TestAuthorizationPreEventListener extends TestCase {
super.setUp();
- int port = MetaStoreUtils.findFreePort();
-
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname,
@@ -62,7 +60,7 @@ public class TestAuthorizationPreEventListener extends TestCase {
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
HadoopDefaultMetastoreAuthenticator.class.getName());
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
clientHiveConf = new HiveConf(this.getClass());
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
index d895da8..312d4aa 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java
@@ -58,13 +58,11 @@ public class TestClientSideAuthorizationProvider extends TestCase {
super.setUp();
- int port = MetaStoreUtils.findFreePort();
-
// Turn off metastore-side authorization
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
"");
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
clientHiveConf = new HiveConf(this.getClass());
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
index 19dc9cf..e11d28c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
@@ -84,8 +84,6 @@ public class TestMetastoreAuthorizationProvider extends TestCase {
super.setUp();
- int port = MetaStoreUtils.findFreePort();
-
// Turn on metastore-side authorization
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
@@ -98,8 +96,7 @@ public class TestMetastoreAuthorizationProvider extends TestCase {
InjectableDummyAuthenticator.class.getName());
System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, "");
-
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
clientHiveConf = createHiveConf();
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
index 5c9bf05..45b443f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
@@ -47,10 +47,6 @@ public class TestMultiAuthorizationPreEventListener {
@BeforeClass
public static void setUp() throws Exception {
-
-
- int port = MetaStoreUtils.findFreePort();
-
System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
AuthorizationPreEventListener.class.getName());
@@ -62,7 +58,7 @@ public class TestMultiAuthorizationPreEventListener {
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
HadoopDefaultMetastoreAuthenticator.class.getName());
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ int port = MetaStoreUtils.startMetaStoreWithRetry();
clientHiveConf = new HiveConf();
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java
index 7148307..a73ea9f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java
@@ -68,13 +68,27 @@ public class ThriftCliServiceMessageSizeTest {
protected static void startHiveServer2WithConf(HiveServer2 hiveServer2, HiveConf hiveConf)
throws Exception {
- hiveServer2.init(hiveConf);
// Start HiveServer2 with given config
// Fail if server doesn't start
- try {
- hiveServer2.start();
- } catch (Throwable t) {
- t.printStackTrace();
+
+ Throwable hs2Exception = null;
+ boolean hs2Started = false;
+ for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) {
+ try {
+ hiveServer2.init(hiveConf);
+ hiveServer2.start();
+ hs2Started = true;
+ break;
+ } catch (Throwable t) {
+ hs2Exception = t;
+ port = MetaStoreUtils.findFreePort();
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port);
+ hiveServer2 = new HiveServer2();
+ }
+ }
+
+ if (!hs2Started) {
+ hs2Exception.printStackTrace();
fail();
}
// Wait for startup to complete
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index ebc4c10..2bf2461 100644
--- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -74,6 +74,7 @@ public class MiniHS2 extends AbstractHiveService {
private final boolean cleanupLocalDirOnStartup;
private final boolean isMetastoreSecure;
private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
+ private boolean usePortsFromConf = false;
public enum MiniClusterType {
MR,
@@ -226,6 +227,7 @@ public class MiniHS2 extends AbstractHiveService {
this.isMetastoreRemote = isMetastoreRemote;
this.isMetastoreSecure = isMetastoreSecure;
this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup;
+ this.usePortsFromConf = usePortsFromConf;
baseDir = getBaseDir();
localFS = FileSystem.getLocal(hiveConf);
FileSystem fs;
@@ -333,19 +335,42 @@ public class MiniHS2 extends AbstractHiveService {
public void start(Map<String, String> confOverlay) throws Exception {
if (isMetastoreRemote) {
- int metaStorePort = MetaStoreUtils.findFreePort();
+ int metaStorePort = MetaStoreUtils.startMetaStoreWithRetry(getHiveConf());
getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
- MetaStoreUtils.startMetaStore(metaStorePort,
- ShimLoader.getHadoopThriftAuthBridge(), getHiveConf());
}
- hiveServer2 = new HiveServer2();
// Set confOverlay parameters
for (Map.Entry<String, String> entry : confOverlay.entrySet()) {
setConfProperty(entry.getKey(), entry.getValue());
}
- hiveServer2.init(getHiveConf());
- hiveServer2.start();
+
+ Exception hs2Exception = null;
+ boolean hs2Started = false;
+ for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) {
+ try {
+ hiveServer2 = new HiveServer2();
+ hiveServer2.init(getHiveConf());
+ hiveServer2.start();
+ hs2Started = true;
+ break;
+ } catch (Exception t) {
+ hs2Exception = t;
+ if (usePortsFromConf) {
+ hs2Started = false;
+ break;
+ } else {
+ HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT,
+ MetaStoreUtils.findFreePort());
+ HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT,
+ MetaStoreUtils.findFreePort());
+ }
+ }
+ }
+
+ if (!hs2Started) {
+ throw(hs2Exception);
+ }
+
waitForStartup();
setStarted(true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 3eb8c36..13bdf8e 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -101,6 +102,8 @@ public class MetaStoreUtils {
public static final String DATABASE_WAREHOUSE_SUFFIX = ".db";
+ public static final int RETRY_COUNT = 10;
+
// Right now we only support one special character '/'.
// More special characters can be added accordingly in the future.
// NOTE:
@@ -1250,6 +1253,36 @@ public class MetaStoreUtils {
return port;
}
+ public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception {
+ return startMetaStoreWithRetry(bridge, null);
+ }
+
+ public static int startMetaStoreWithRetry(HiveConf conf) throws Exception {
+ return startMetaStoreWithRetry(ShimLoader.getHadoopThriftAuthBridge(), conf);
+ }
+
+ public static int startMetaStoreWithRetry() throws Exception {
+ return startMetaStoreWithRetry(ShimLoader.getHadoopThriftAuthBridge(), null);
+ }
+
+ public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf)
+ throws Exception {
+ Exception metaStoreException = null;
+ int metaStorePort = 0;
+
+ for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) {
+ try {
+ metaStorePort = findFreePort();
+ startMetaStore(metaStorePort, bridge, conf);
+ return metaStorePort;
+ } catch (ConnectException ce) {
+ metaStoreException = ce;
+ }
+ }
+
+ throw metaStoreException;
+ }
+
public static int startMetaStore(HiveConf conf) throws Exception {
return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1f7e3be0/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
index 6fec947..8977a96 100644
--- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
+++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java
@@ -95,15 +95,30 @@ public class ThriftCliServiceTestWithCookie {
}
protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception {
- hiveServer2.init(hiveConf);
// Start HiveServer2 with given config
// Fail if server doesn't start
- try {
- hiveServer2.start();
- } catch (Throwable t) {
- t.printStackTrace();
+
+ Throwable hs2Exception = null;
+ boolean hs2Started = false;
+ for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) {
+ try {
+ hiveServer2.init(hiveConf);
+ hiveServer2.start();
+ hs2Started = true;
+ break;
+ } catch (Throwable t) {
+ hs2Exception = t;
+ port = MetaStoreUtils.findFreePort();
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port);
+ hiveServer2 = new HiveServer2();
+ }
+ }
+
+ if (!hs2Started) {
+ hs2Exception.printStackTrace();
fail();
}
+
// Wait for startup to complete
Thread.sleep(2000);
System.out.println("HiveServer2 started on port " + port);