kudu git commit: KUDU-1681: DNS resolution failure of master hostname causes tserver crash

[da...@apache.org]

Repository: kudu
Updated Branches:
  refs/heads/master cae2f9899 -> fc5e6a3e7


KUDU-1681: DNS resolution failure of master hostname causes tserver crash

Also changes the fail_dns_resolution flag to make DNS resolution return
an error Status instead of an empty resolved list. From the manpage of
getaddrinfo:

    After a successful call to getaddrinfo(), *res is a pointer to a
    linked list of one or more addrinfo structures.

Change-Id: Ib79973a38bebbe84df419e274c08c8cc0c5d31cd
Reviewed-on: http://gerrit.cloudera.org:8080/4640
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fc5e6a3e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fc5e6a3e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fc5e6a3e

Branch: refs/heads/master
Commit: fc5e6a3e7921b9a475848117063e2d7db56f8ee4
Parents: cae2f98
Author: Dan Burkert <da...@cloudera.com>
Authored: Wed Oct 5 18:40:17 2016 -0700
Committer: Dan Burkert <da...@cloudera.com>
Committed: Thu Oct 6 19:16:41 2016 +0000

----------------------------------------------------------------------
 src/kudu/tserver/heartbeater.cc        |  4 +---
 src/kudu/tserver/tablet_server-test.cc | 13 ++++++++++++-
 src/kudu/util/net/net_util.cc          |  2 +-
 3 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/fc5e6a3e/src/kudu/tserver/heartbeater.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/heartbeater.cc b/src/kudu/tserver/heartbeater.cc
index 231539b..f16214c 100644
--- a/src/kudu/tserver/heartbeater.cc
+++ b/src/kudu/tserver/heartbeater.cc
@@ -280,9 +280,7 @@ Heartbeater::Thread::Thread(const HostPort& master_address, TabletServer* server
 
 Status Heartbeater::Thread::ConnectToMaster() {
   gscoped_ptr<MasterServiceProxy> new_proxy;
-  MasterServiceProxyForHostPort(master_address_,
-                                server_->messenger(),
-                                &new_proxy);
+  RETURN_NOT_OK(MasterServiceProxyForHostPort(master_address_, server_->messenger(), &new_proxy));
 
   // Ping the master to verify that it's alive.
   master::PingRequestPB req;

http://git-wip-us.apache.org/repos/asf/kudu/blob/fc5e6a3e/src/kudu/tserver/tablet_server-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_server-test.cc b/src/kudu/tserver/tablet_server-test.cc
index 6d99c24..76ddad6 100644
--- a/src/kudu/tserver/tablet_server-test.cc
+++ b/src/kudu/tserver/tablet_server-test.cc
@@ -29,6 +29,7 @@
 #include "kudu/server/server_base.pb.h"
 #include "kudu/server/server_base.proxy.h"
 #include "kudu/tablet/tablet_bootstrap.h"
+#include "kudu/tserver/heartbeater.h"
 #include "kudu/util/crc.h"
 #include "kudu/util/curl_util.h"
 #include "kudu/util/url-coding.h"
@@ -56,8 +57,9 @@ DEFINE_int32(single_threaded_insert_latency_bench_insert_rows, 1000,
              "Number of rows to insert in the testing phase of the single threaded"
              " tablet server insert latency micro-benchmark");
 
-DECLARE_int32(scanner_batch_size_rows);
+DECLARE_bool(fail_dns_resolution);
 DECLARE_int32(metrics_retirement_age_ms);
+DECLARE_int32(scanner_batch_size_rows);
 DECLARE_string(block_manager);
 
 // Declare these metrics prototypes for simpler unit testing of their behavior.
@@ -2335,5 +2337,14 @@ TEST_F(TabletServerTest, TestKudu120PreRequisites) {
   flush_done_latch.Wait();
 }
 
+// Test DNS resolution failure in the master heartbeater.
+// Regression test for KUDU-1681.
+TEST_F(TabletServerTest, TestFailedDnsResolution) {
+  FLAGS_fail_dns_resolution = true;
+  mini_server_->server()->heartbeater()->TriggerASAP();
+  // Wait to make sure the heartbeater thread attempts the DNS lookup.
+  usleep(100 * 1000);
+}
+
 } // namespace tserver
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/fc5e6a3e/src/kudu/util/net/net_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/net/net_util.cc b/src/kudu/util/net/net_util.cc
index 0564398..ac5fb62 100644
--- a/src/kudu/util/net/net_util.cc
+++ b/src/kudu/util/net/net_util.cc
@@ -142,7 +142,7 @@ Status HostPort::ResolveAddresses(vector<Sockaddr>* addresses) const {
             << " for host/port " << ToString();
   }
   if (PREDICT_FALSE(FLAGS_fail_dns_resolution)) {
-    addresses->clear();
+    return Status::NetworkError("injected DNS resolution failure");
   }
   return Status::OK();
 }