You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/04/12 23:18:46 UTC
[12/50] incubator-impala git commit: IMPALA-2888:
StatestoreSslTest.SmokeTest failed
IMPALA-2888: StatestoreSslTest.SmokeTest failed
The StatestoreTest.SmokeTest and StatestoreSslTest.SmokeTest both use
ephemeral ports for the statestore subscribers without checking first
if they are unused or not. This is most likely the reason behind these
tests failing.
This patch includes a new util function which checks for and returns
unused ephemeral ports, which the tests now use.
Change-Id: I51b7a2c609d3a5ae6f7d32ef38db89efe71a882b
Reviewed-on: http://gerrit.cloudera.org:8080/2630
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/2b47c5a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/2b47c5a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/2b47c5a3
Branch: refs/heads/master
Commit: 2b47c5a3bf454aca199cf0a9b2397b540b6cdf0d
Parents: b2ccb17
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Fri Mar 25 10:22:53 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Mar 29 02:04:02 2016 +0000
----------------------------------------------------------------------
be/src/statestore/statestore-test.cc | 17 ++++++++++----
be/src/testutil/in-process-servers.cc | 37 +++++++++++++++++++-----------
be/src/util/network-util.cc | 27 ++++++++++++++++++++++
be/src/util/network-util.h | 3 +++
4 files changed, 66 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2b47c5a3/be/src/statestore/statestore-test.cc
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestore-test.cc b/be/src/statestore/statestore-test.cc
index 2338f8b..d0b7d88 100644
--- a/be/src/statestore/statestore-test.cc
+++ b/be/src/statestore/statestore-test.cc
@@ -41,14 +41,17 @@ TEST(StatestoreTest, SmokeTest) {
new InProcessStatestore(ips->port(), ips->port() + 10);
ASSERT_FALSE(statestore_wont_start->Start().ok());
+ int subscriber_port = FindUnusedEphemeralPort();
+ ASSERT_NE(subscriber_port, -1) << "Could not find unused port";
+
StatestoreSubscriber* sub_will_start = new StatestoreSubscriber("sub1",
- MakeNetworkAddress("localhost", ips->port() + 20),
+ MakeNetworkAddress("localhost", subscriber_port),
MakeNetworkAddress("localhost", ips->port()), new MetricGroup(""));
ASSERT_OK(sub_will_start->Start());
// Confirm that a subscriber trying to use an in-use port will fail to start.
StatestoreSubscriber* sub_will_not_start = new StatestoreSubscriber("sub2",
- MakeNetworkAddress("localhost", ips->port() + 20),
+ MakeNetworkAddress("localhost", subscriber_port),
MakeNetworkAddress("localhost", ips->port()), new MetricGroup(""));
ASSERT_FALSE(sub_will_not_start->Start().ok());
}
@@ -66,16 +69,22 @@ TEST(StatestoreSslTest, SmokeTest) {
InProcessStatestore* statestore = InProcessStatestore::StartWithEphemeralPorts();
if (statestore == NULL) FAIL() << "Unable to start Statestore";
+ int subscriber_port = FindUnusedEphemeralPort();
+ ASSERT_NE(subscriber_port, -1) << "Could not find unused port";
+
StatestoreSubscriber* sub_will_start = new StatestoreSubscriber("smoke_sub1",
- MakeNetworkAddress("localhost", statestore->port() + 10),
+ MakeNetworkAddress("localhost", subscriber_port),
MakeNetworkAddress("localhost", statestore->port()), new MetricGroup(""));
ASSERT_OK(sub_will_start->Start());
stringstream invalid_server_cert;
invalid_server_cert << impala_home << "/be/src/testutil/invalid-server-cert.pem";
FLAGS_ssl_client_ca_certificate = invalid_server_cert.str();
+ int another_subscriber_port = FindUnusedEphemeralPort();
+ ASSERT_NE(another_subscriber_port, -1) << "Could not find unused port";
+
StatestoreSubscriber* sub_will_not_start = new StatestoreSubscriber("smoke_sub2",
- MakeNetworkAddress("localhost", statestore->port() + 20),
+ MakeNetworkAddress("localhost", another_subscriber_port),
MakeNetworkAddress("localhost", statestore->port()), new MetricGroup(""));
ASSERT_FALSE(sub_will_not_start->Start().ok());
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2b47c5a3/be/src/testutil/in-process-servers.cc
----------------------------------------------------------------------
diff --git a/be/src/testutil/in-process-servers.cc b/be/src/testutil/in-process-servers.cc
index 016aece..a502f26 100644
--- a/be/src/testutil/in-process-servers.cc
+++ b/be/src/testutil/in-process-servers.cc
@@ -36,19 +36,24 @@ using namespace apache::thrift;
using namespace impala;
using boost::shared_ptr;
-/// Pick a random port in the range of ephemeral ports
-/// https://tools.ietf.org/html/rfc6335
-static uint32_t RandomEphemeralPort() {
- static uint32_t LOWER = 49152, UPPER = 65000;
- return LOWER + rand() % (UPPER - LOWER);
-}
-
InProcessImpalaServer* InProcessImpalaServer::StartWithEphemeralPorts(
const string& statestore_host, int statestore_port) {
- for (uint32_t tries = 0; tries < 10; ++tries) {
- uint32_t p = RandomEphemeralPort();
- uint32_t backend_port = p, subscriber_port = ++p, webserver_port = ++p,
- beeswax_port = ++p, hs2_port = ++p;
+ for (int tries = 0; tries < 10; ++tries) {
+ int backend_port = FindUnusedEphemeralPort();
+ if (backend_port == -1) continue;
+
+ int subscriber_port = FindUnusedEphemeralPort();
+ if (subscriber_port == -1) continue;
+
+ int webserver_port = FindUnusedEphemeralPort();
+ if (webserver_port == -1) continue;
+
+ int beeswax_port = FindUnusedEphemeralPort();
+ if (beeswax_port == -1) continue;
+
+ int hs2_port = FindUnusedEphemeralPort();
+ if (hs2_port == -1) continue;
+
InProcessImpalaServer* impala =
new InProcessImpalaServer("localhost", backend_port, subscriber_port,
webserver_port, statestore_host, statestore_port);
@@ -122,9 +127,13 @@ Status InProcessImpalaServer::Join() {
}
InProcessStatestore* InProcessStatestore::StartWithEphemeralPorts() {
- for (uint32_t tries = 0; tries < 10; ++tries) {
- uint32_t p = RandomEphemeralPort();
- uint32_t statestore_port = p, webserver_port = ++p;
+ for (int tries = 0; tries < 10; ++tries) {
+ int statestore_port = FindUnusedEphemeralPort();
+ if (statestore_port == -1) continue;
+
+ int webserver_port = FindUnusedEphemeralPort();
+ if (webserver_port == -1) continue;
+
InProcessStatestore* ips = new InProcessStatestore(statestore_port, webserver_port);
if (ips->Start().ok()) return ips;
delete ips;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2b47c5a3/be/src/util/network-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/network-util.cc b/be/src/util/network-util.cc
index ea18110..edfc953 100644
--- a/be/src/util/network-util.cc
+++ b/be/src/util/network-util.cc
@@ -32,6 +32,7 @@
using boost::algorithm::is_any_of;
using boost::algorithm::split;
+using std::random_device;
#ifdef __APPLE__
// OS X does not seem to have a similar limitation as Linux and thus the
@@ -140,4 +141,30 @@ ostream& operator<<(ostream& out, const TNetworkAddress& hostport) {
return out;
}
+/// Pick a random port in the range of ephemeral ports
+/// https://tools.ietf.org/html/rfc6335
+int FindUnusedEphemeralPort() {
+ static uint32_t LOWER = 49152, UPPER = 65000;
+ random_device rd;
+ srand(rd());
+
+ int sockfd = socket(AF_INET, SOCK_STREAM, 0);
+ if (sockfd < 0) return -1;
+ struct sockaddr_in server_address;
+ bzero(reinterpret_cast<char*>(&server_address), sizeof(server_address));
+ server_address.sin_family = AF_INET;
+ server_address.sin_addr.s_addr = INADDR_ANY;
+ for (uint32_t tries = 0; tries < 10; ++tries) {
+ int port = LOWER + rand() % (UPPER - LOWER);
+ server_address.sin_port = htons(port);
+ if (bind(sockfd, reinterpret_cast<struct sockaddr*>(&server_address),
+ sizeof(server_address)) == 0) {
+ close(sockfd);
+ return port;
+ }
+ }
+ close(sockfd);
+ return -1;
+}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2b47c5a3/be/src/util/network-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/network-util.h b/be/src/util/network-util.h
index 4e923d7..e2cb88a 100644
--- a/be/src/util/network-util.h
+++ b/be/src/util/network-util.h
@@ -49,4 +49,7 @@ std::string TNetworkAddressToString(const TNetworkAddress& address);
/// Prints a hostport as ipaddress:port
std::ostream& operator<<(std::ostream& out, const TNetworkAddress& hostport);
+/// Returns a ephemeral port that is unused when this function executes. Returns -1 on an
+/// error or if a free ephemeral port can't be found after 10 tries.
+int FindUnusedEphemeralPort();
}