You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by jd...@apache.org on 2016/04/28 04:32:37 UTC

[1/5] incubator-kudu git commit: [java client] Add a way to restart processes

Repository: incubator-kudu
Updated Branches:
  refs/heads/master 73b8cc07e -> c2aed1b7c


[java client] Add a way to restart processes

This patch adds more tracking of the processes started by MiniKuduCluster.
Mainly we needed to record the command lines used to start the masters
and tablet servers.

This patch adds a new unit test for the mini cluster, before we only relied
on the fact that everything was using it so therefore it was tested.

Finally getUniqueLocalhost was moved to TestUtils so that findFreePort
could also use it. The reason is that the processes on OSX were binding
to 0.0.0.0 but findFreePort was using a different hostnames in
ServerSocket and so the behavior was broken. This was noticed when a TS
leaked after a test and new tests were failing because the TS was already
bound to a port we thought to be free.

Change-Id: I065917cd219e1dd8fa93fed9ca36d58a3372935d
Reviewed-on: http://gerrit.cloudera.org:8080/2828
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Dan Burkert <da...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/39441221
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/39441221
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/39441221

Branch: refs/heads/master
Commit: 39441221a0f847c547df691d60e11860e4052b71
Parents: 73b8cc0
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Wed Apr 20 15:57:20 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Apr 28 01:48:40 2016 +0000

----------------------------------------------------------------------
 .../java/org/kududb/client/BaseKuduTest.java    |  76 ++++++++---
 .../java/org/kududb/client/MiniKuduCluster.java | 129 +++++++++++--------
 .../org/kududb/client/TestMiniKuduCluster.java  | 116 +++++++++++++++++
 .../test/java/org/kududb/client/TestUtils.java  |  60 ++++++++-
 4 files changed, 300 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/39441221/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
index 996c746..d92a612 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
@@ -16,30 +16,29 @@
 // under the License.
 package org.kududb.client;
 
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.kududb.ColumnSchema;
 import org.kududb.Schema;
 import org.kududb.Type;
 import org.kududb.master.Master;
-import org.kududb.util.NetUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.fail;
 
 public class BaseKuduTest {
 
@@ -54,22 +53,25 @@ public class BaseKuduTest {
   private static final int NUM_MASTERS =
       Integer.getInteger(NUM_MASTERS_PROP, DEFAULT_NUM_MASTERS);
 
+  protected static final int DEFAULT_SLEEP = 50000;
+
+  // Currently not specifying a seed since we want a random behavior when running tests that
+  // restart tablet servers. Would be nice to have the same kind of facility that C++ has that dumps
+  // the seed it picks so that you can re-run tests with it.
+  private static final Random randomForTSRestart = new Random();
+
   private static MiniKuduCluster miniCluster;
 
   // Comma separate describing the master addresses and ports.
   protected static String masterAddresses;
   protected static List<HostAndPort> masterHostPorts;
 
-  protected static final int DEFAULT_SLEEP = 50000;
-
   // We create both versions of the client for ease of use.
   protected static AsyncKuduClient client;
   protected static KuduClient syncClient;
   protected static Schema basicSchema = getBasicSchema();
   protected static Schema allTypesSchema = getSchemaWithAllTypes();
 
-  private static List<String> tableNames = new ArrayList<>();
-
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     LOG.info("Setting up before class...");
@@ -133,7 +135,6 @@ public class BaseKuduTest {
       fail("Got error during table creation, is the Kudu master running at " +
           masterAddresses + "?");
     }
-    tableNames.add(tableName);
     return table;
   }
 
@@ -355,6 +356,43 @@ public class BaseKuduTest {
   }
 
   /**
+   * Picks at random a tablet server that serves tablets from the passed table and restarts it.
+   * Waits between killing and restarting the process.
+   * @param table table to query for a TS to restart
+   * @throws Exception
+   */
+  protected static void restartTabletServer(KuduTable table) throws Exception {
+    List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
+    if (tablets.isEmpty()) {
+      fail("Table " + table.getName() + " doesn't have any tablets");
+    }
+
+    LocatedTablet tablet = tablets.get(0);
+
+    int port = tablet.getReplicas().get(
+        randomForTSRestart.nextInt(tablet.getReplicas().size())).getRpcPort();
+
+    miniCluster.killTabletServerOnPort(port);
+
+    Thread.sleep(1000);
+
+    miniCluster.restartDeadTabletServerOnPort(port);
+  }
+
+  /**
+   * Kills, sleeps, then restarts the leader master.
+   * @throws Exception
+   */
+  protected static void restartLeaderMaster() throws Exception {
+    int master = findLeaderMasterPort();
+    miniCluster.killMasterOnPort(master);
+
+    Thread.sleep(1000);
+
+    miniCluster.restartDeadMasterOnPort(master);
+  }
+
+  /**
    * Return the comma-separated list of "host:port" pairs that describes the master
    * config for this cluster.
    * @return The master config string.

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/39441221/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
index 9bd5442..00a7255 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
@@ -13,6 +13,7 @@
  */
 package org.kududb.client;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
@@ -27,17 +28,13 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-import java.lang.reflect.Method;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import sun.management.VMManagement;
-
 /**
  * Utility class to start and manipulate Kudu clusters. Relies on being IN the Kudu source code with
  * both the kudu-master and kudu-tserver binaries already compiled. {@link BaseKuduTest} should be
@@ -59,6 +56,9 @@ public class MiniKuduCluster implements AutoCloseable {
   // Map of ports to tablet servers.
   private final Map<Integer, Process> tserverProcesses = new ConcurrentHashMap<>();
 
+  // Map of ports to process command lines. Never removed from. Used to restart processes.
+  private final Map<Integer, String[]> commandLines = new ConcurrentHashMap<>();
+
   private final List<String> pathsToDelete = new ArrayList<>();
   private final List<HostAndPort> masterHostPorts = new ArrayList<>();
 
@@ -96,45 +96,6 @@ public class MiniKuduCluster implements AutoCloseable {
   }
 
   /**
-   * @return the local PID of this process.
-   * This is used to generate unique loopback IPs for parallel test running.
-   */
-  private static int getPid() {
-    try {
-      RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
-      java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm");
-      jvm.setAccessible(true);
-      VMManagement mgmt = (VMManagement)jvm.get(runtime);
-      Method pid_method = mgmt.getClass().getDeclaredMethod("getProcessId");
-      pid_method.setAccessible(true);
-
-      return (Integer)pid_method.invoke(mgmt);
-    } catch (Exception e) {
-      LOG.warn("Cannot get PID", e);
-      return 1;
-    }
-  }
-
-  /**
-   * @return a unique loopback IP address for this PID. This allows running
-   * tests in parallel, since 127.0.0.0/8 all act as loopbacks on Linux.
-   *
-   * The generated IP is based on pid, so this requires that the parallel tests
-   * run in separate VMs.
-   *
-   * On OSX, the above trick doesn't work, so we can't run parallel tests on OSX.
-   * Given that, we just return the normal localhost IP.
-   */
-  private static String getUniqueLocalhost() {
-    if ("Mac OS X".equals(System.getProperty("os.name"))) {
-      return "127.0.0.1";
-    }
-
-    int pid = getPid();
-    return "127." + ((pid & 0xff00) >> 8) + "." + (pid & 0xff) + ".1";
-  }
-
-  /**
    * Starts a Kudu cluster composed of the provided masters and tablet servers.
    * @param numMasters how many masters to start
    * @param numTservers how many tablet servers to start
@@ -145,14 +106,14 @@ public class MiniKuduCluster implements AutoCloseable {
     Preconditions.checkArgument(numTservers > 0, "Need at least one tablet server");
     // The following props are set via kudu-client's pom.
     String baseDirPath = TestUtils.getBaseDir();
-    String localhost = getUniqueLocalhost();
+    String localhost = TestUtils.getUniqueLocalhost();
 
     long now = System.currentTimeMillis();
     LOG.info("Starting {} masters...", numMasters);
-    int port = startMasters(PORT_START, numMasters, baseDirPath);
+    int startPort = startMasters(PORT_START, numMasters, baseDirPath);
     LOG.info("Starting {} tablet servers...", numTservers);
+    List<Integer> ports = TestUtils.findFreePorts(startPort, numTservers * 2);
     for (int i = 0; i < numTservers; i++) {
-      port = TestUtils.findFreePort(port);
       String dataDirPath = baseDirPath + "/ts-" + i + "-" + now;
       String flagsPath = TestUtils.getFlagsPath();
       String[] tsCmdLine = {
@@ -163,9 +124,10 @@ public class MiniKuduCluster implements AutoCloseable {
           "--tserver_master_addrs=" + masterAddresses,
           "--webserver_interface=" + localhost,
           "--local_ip_for_outbound_sockets=" + localhost,
-          "--rpc_bind_addresses=" + localhost + ":" + port};
-      tserverProcesses.put(port, configureAndStartProcess(tsCmdLine));
-      port++;
+          "--webserver_port=" + ports.get((i * 2) + 1),
+          "--rpc_bind_addresses=" + localhost + ":" + ports.get(i * 2)};
+      tserverProcesses.put(ports.get(i * 2), configureAndStartProcess(tsCmdLine));
+      commandLines.put(ports.get(i * 2), tsCmdLine);
 
       if (flagsPath.startsWith(baseDirPath)) {
         // We made a temporary copy of the flags; delete them later.
@@ -191,7 +153,7 @@ public class MiniKuduCluster implements AutoCloseable {
     // Get the list of web and RPC ports to use for the master consensus configuration:
     // request NUM_MASTERS * 2 free ports as we want to also reserve the web
     // ports for the consensus configuration.
-    String localhost = getUniqueLocalhost();
+    String localhost = TestUtils.getUniqueLocalhost();
     List<Integer> ports = TestUtils.findFreePorts(masterStartPort, numMasters * 2);
     int lastFreePort = ports.get(ports.size() - 1);
     List<Integer> masterRpcPorts = Lists.newArrayListWithCapacity(numMasters);
@@ -205,8 +167,9 @@ public class MiniKuduCluster implements AutoCloseable {
       }
     }
     masterAddresses = NetUtil.hostsAndPortsToString(masterHostPorts);
+    long now = System.currentTimeMillis();
     for (int i = 0; i < numMasters; i++) {
-      long now = System.currentTimeMillis();
+      int port = masterRpcPorts.get(i);
       String dataDirPath = baseDirPath + "/master-" + i + "-" + now;
       String flagsPath = TestUtils.getFlagsPath();
       // The web port must be reserved in the call to findFreePorts above and specified
@@ -223,13 +186,14 @@ public class MiniKuduCluster implements AutoCloseable {
           "--fs_data_dirs=" + dataDirPath,
           "--webserver_interface=" + localhost,
           "--local_ip_for_outbound_sockets=" + localhost,
-          "--rpc_bind_addresses=" + localhost + ":" + masterRpcPorts.get(i),
+          "--rpc_bind_addresses=" + localhost + ":" + port,
           "--webserver_port=" + masterWebPorts.get(i));
       if (numMasters > 1) {
         masterCmdLine.add("--master_addresses=" + masterAddresses);
       }
-      masterProcesses.put(masterRpcPorts.get(i),
-          configureAndStartProcess(masterCmdLine.toArray(new String[masterCmdLine.size()])));
+      String[] commandLine = masterCmdLine.toArray(new String[masterCmdLine.size()]);
+      masterProcesses.put(port, configureAndStartProcess(commandLine));
+      commandLines.put(port, commandLine);
 
       if (flagsPath.startsWith(baseDirPath)) {
         // We made a temporary copy of the flags; delete them later.
@@ -275,6 +239,43 @@ public class MiniKuduCluster implements AutoCloseable {
   }
 
   /**
+   * Starts a previously killed master process on the specified port.
+   * @param port which port the master was listening on for RPCs
+   * @return true if it was successful, false otherwise
+   * @throws Exception
+   */
+  public void restartDeadMasterOnPort(int port) throws Exception {
+    restartDeadProcessOnPort(port, masterProcesses);
+  }
+
+  /**
+   * Starts a previously killed tablet server process on the specified port.
+   * @param port which port the TS was listening on for RPCs
+   * @return true if it was successful, false otherwise
+   * @throws Exception
+   */
+  public void restartDeadTabletServerOnPort(int port) throws Exception {
+    restartDeadProcessOnPort(port, tserverProcesses);
+  }
+
+  private void restartDeadProcessOnPort(int port, Map<Integer, Process> map) throws Exception {
+    if (!commandLines.containsKey(port)) {
+      String message = "Cannot start process on unknown port " + port;
+      LOG.warn(message);
+      throw new RuntimeException(message);
+    }
+
+    if (map.containsKey(port)) {
+      String message = "Process already exists on port " + port;
+      LOG.warn(message);
+      throw new RuntimeException(message);
+    }
+
+    String[] commandLine = commandLines.get(port);
+    map.put(port, configureAndStartProcess(commandLine));
+  }
+
+  /**
    * Kills the TS listening on the provided port. Doesn't do anything if the TS was already killed.
    * @param port port on which the tablet server is listening on
    * @throws InterruptedException
@@ -363,6 +364,24 @@ public class MiniKuduCluster implements AutoCloseable {
   }
 
   /**
+   * Returns an unmodifiable map of all tablet servers in pairs of RPC port - > Process.
+   * @return an unmodifiable map of all tablet servers
+   */
+  @VisibleForTesting
+  Map<Integer, Process> getTabletServerProcesses() {
+    return Collections.unmodifiableMap(tserverProcesses);
+  }
+
+  /**
+   * Returns an unmodifiable map of all masters in pairs of RPC port - > Process.
+   * @return an unmodifiable map of all masters
+   */
+  @VisibleForTesting
+  Map<Integer, Process> getMasterProcesses() {
+    return Collections.unmodifiableMap(masterProcesses);
+  }
+
+  /**
    * Helper runnable that can log what the processes are sending on their stdout and stderr that
    * we'd otherwise miss.
    */

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/39441221/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java b/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java
new file mode 100644
index 0000000..82ffacb
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.client;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.Socket;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestMiniKuduCluster {
+
+  private static final int NUM_TABLET_SERVERS = 3;
+  private static final int DEFAULT_NUM_MASTERS = 1;
+
+  private MiniKuduCluster cluster;
+
+  @Before
+  public void before() throws Exception {
+    cluster = new MiniKuduCluster.MiniKuduClusterBuilder()
+        .numMasters(DEFAULT_NUM_MASTERS)
+        .numTservers(NUM_TABLET_SERVERS)
+        .build();
+    assertTrue(cluster.waitForTabletServers(NUM_TABLET_SERVERS));
+  }
+
+  @After
+  public void after() {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Test(timeout = 50000)
+  public void test() throws Exception {
+
+    assertEquals(DEFAULT_NUM_MASTERS, cluster.getMasterProcesses().size());
+    assertEquals(NUM_TABLET_SERVERS, cluster.getTabletServerProcesses().size());
+
+    {
+      // Kill the master.
+      int masterPort = cluster.getMasterProcesses().keySet().iterator().next();
+      testPort(masterPort, true, 1000);
+      cluster.killMasterOnPort(masterPort);
+
+      testPort(masterPort, false, 2000);
+
+      // Restart the master.
+      cluster.restartDeadMasterOnPort(masterPort);
+
+      // Test we can reach it.
+      testPort(masterPort, true, 3000);
+    }
+
+
+    {
+      // Kill the first TS.
+      int tsPort = cluster.getTabletServerProcesses().keySet().iterator().next();
+      testPort(tsPort, true, 1000);
+      cluster.killTabletServerOnPort(tsPort);
+
+      testPort(tsPort, false, 2000);
+
+      // Restart it.
+      cluster.restartDeadTabletServerOnPort(tsPort);
+
+      testPort(tsPort, true, 3000);
+    }
+
+    assertEquals(DEFAULT_NUM_MASTERS, cluster.getMasterProcesses().size());
+    assertEquals(NUM_TABLET_SERVERS, cluster.getTabletServerProcesses().size());
+  }
+
+  /**
+   * Test without the specified is open or closed, waiting up to a certain time.
+   * The longer you expect it might for the socket to become open or closed.
+   * @param port the port to test
+   * @param testIsOpen true if we should want it to be open, false if we want it closed
+   * @param timeout how long we're willing to wait before it happens
+   * @throws InterruptedException
+   */
+  private void testPort(int port, boolean testIsOpen, long timeout) throws InterruptedException {
+    DeadlineTracker tracker = new DeadlineTracker();
+    while (tracker.getElapsedMillis() < timeout) {
+      try {
+        Socket socket = new Socket(TestUtils.getUniqueLocalhost(), port);
+        socket.close();
+        if (testIsOpen) {
+          return;
+        }
+      } catch (IOException e) {
+        if (!testIsOpen) {
+          return;
+        }
+      }
+      Thread.sleep(200);
+    }
+    fail("Port " + port + " is still " + (testIsOpen ? "closed " : "open"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/39441221/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java b/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
index c40cda4..392bed9 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
@@ -20,13 +20,21 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.sun.security.auth.module.UnixSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sun.management.VMManagement;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
 import java.net.ServerSocket;
+import java.net.SocketAddress;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.nio.file.Files;
@@ -36,9 +44,6 @@ import java.nio.file.StandardCopyOption;
 import java.util.List;
 import java.util.Set;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 /**
  * A grouping of methods that help unit testing.
  */
@@ -143,16 +148,18 @@ public class TestUtils {
    * Finds the next free port, starting with the one passed. Keep in mind the
    * time-of-check-time-of-use nature of this method, the returned port might become occupied
    * after it was checked for availability.
-   * @param startPort First port to be probed.
-   * @return A currently usable port.
+   * @param startPort first port to be probed
+   * @return a currently usable port
    * @throws IOException IOE is thrown if we can't close a socket we tried to open or if we run
-   * out of ports to try.
+   * out of ports to try
    */
   public static int findFreePort(int startPort) throws IOException {
     ServerSocket ss;
     for(int i = startPort; i < 65536; i++) {
       try {
-        ss = new ServerSocket(i);
+        ss = new ServerSocket();
+        SocketAddress address = new InetSocketAddress(getUniqueLocalhost(), i);
+        ss.bind(address);
       } catch (IOException e) {
         continue;
       }
@@ -240,4 +247,43 @@ public class TestUtils {
   static void resumeProcess(Process proc) throws Exception {
     signalProcess(proc, "CONT");
   }
+
+  /**
+   * This is used to generate unique loopback IPs for parallel test running.
+   * @return the local PID of this process
+   */
+  static int getPid() {
+    try {
+      RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
+      java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm");
+      jvm.setAccessible(true);
+      VMManagement mgmt = (VMManagement)jvm.get(runtime);
+      Method pid_method = mgmt.getClass().getDeclaredMethod("getProcessId");
+      pid_method.setAccessible(true);
+
+      return (Integer)pid_method.invoke(mgmt);
+    } catch (Exception e) {
+      LOG.warn("Cannot get PID", e);
+      return 1;
+    }
+  }
+
+  /**
+   * The generated IP is based on pid, so this requires that the parallel tests
+   * run in separate VMs.
+   *
+   * On OSX, the above trick doesn't work, so we can't run parallel tests on OSX.
+   * Given that, we just return the normal localhost IP.
+   *
+   * @return a unique loopback IP address for this PID. This allows running
+   * tests in parallel, since 127.0.0.0/8 all act as loopbacks on Linux.
+   */
+  static String getUniqueLocalhost() {
+    if ("Mac OS X".equals(System.getProperty("os.name"))) {
+      return "127.0.0.1";
+    }
+
+    int pid = getPid();
+    return "127." + ((pid & 0xff00) >> 8) + "." + (pid & 0xff) + ".1";
+  }
 }


[3/5] incubator-kudu git commit: Bump up some default timeouts

Posted by jd...@apache.org.
Bump up some default timeouts

The clients have longer default timeouts. For Java, the default operation timeout and the default
admin timeout are now set to 30 seconds instead of 10. The default socket read timeout is now 10
seconds instead of 5. For the C++ client, the default admin timeout is now 30 seconds instead of
10, the default RPC timeout is now 10 seconds instead of 5, and the default scan timeout is now
30 seconds instead of 15.

Change-Id: I49d2faced9011256e7ec7ab16241967bee6c6dd9
Reviewed-on: http://gerrit.cloudera.org:8080/2872
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/01e8329c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/01e8329c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/01e8329c

Branch: refs/heads/master
Commit: 01e8329cfaf4384377b1499ea6be0a04adc92631
Parents: c4180bb
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Tue Apr 26 11:20:45 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Apr 28 02:31:25 2016 +0000

----------------------------------------------------------------------
 docs/release_notes.adoc                                   |  9 +++++++++
 .../src/main/java/org/kududb/client/AsyncKuduClient.java  | 10 +++++-----
 .../src/main/java/org/kududb/client/KuduClient.java       |  6 +++---
 .../src/test/java/org/kududb/client/ITClient.java         |  1 -
 src/kudu/client/client-test.cc                            |  4 +---
 src/kudu/client/client.h                                  |  6 +++---
 src/kudu/client/client_builder-internal.cc                |  4 ++--
 src/kudu/client/predicate-test.cc                         |  1 -
 src/kudu/integration-tests/all_types-itest.cc             |  1 -
 9 files changed, 23 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/docs/release_notes.adoc
----------------------------------------------------------------------
diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc
index 1f0a3d3..4c73cb4 100644
--- a/docs/release_notes.adoc
+++ b/docs/release_notes.adoc
@@ -75,6 +75,15 @@ Hadoop storage technologies.
   using mapreduce api. Includes predicate pushdowns for spark-sql and spark filters.
   Parallel retrieval for multiple tablets and column projections. link:developing.html#_kudu_integration_with_spark[Kudu integration with Spark Example]
 
+[[rn_0.9.0_changes]]
+==== Other noteworthy changes
+
+- The clients have longer default timeouts. For Java, the default operation timeout and the default
+  admin operation timeout are now set to 30 seconds instead of 10. The default socket read timeout
+  is now 10 seconds instead of 5. For the C++ client, the default admin timeout is now 30 seconds
+  instead of 10, the default RPC timeout is now 10 seconds instead of 5, and the default scan
+  timeout is now 30 seconds instead of 15.
+
 [[rn_0.8.0]]
 === Release notes specific to 0.8.0
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
index 6482996..5483288 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
@@ -126,8 +126,8 @@ public class AsyncKuduClient implements AutoCloseable {
   public static final int SLEEP_TIME = 500;
   public static final byte[] EMPTY_ARRAY = new byte[0];
   public static final long NO_TIMESTAMP = -1;
-  public static final long DEFAULT_OPERATION_TIMEOUT_MS = 10000;
-  public static final long DEFAULT_SOCKET_READ_TIMEOUT_MS = 5000;
+  public static final long DEFAULT_OPERATION_TIMEOUT_MS = 30000;
+  public static final long DEFAULT_SOCKET_READ_TIMEOUT_MS = 10000;
 
   private final ClientSocketChannelFactory channelFactory;
 
@@ -2101,7 +2101,7 @@ public class AsyncKuduClient implements AutoCloseable {
      * Sets the default timeout used for administrative operations (e.g. createTable, deleteTable,
      * etc).
      * Optional.
-     * If not provided, defaults to 10s.
+     * If not provided, defaults to 30s.
      * A value of 0 disables the timeout.
      * @param timeoutMs a timeout in milliseconds
      * @return this builder
@@ -2114,7 +2114,7 @@ public class AsyncKuduClient implements AutoCloseable {
     /**
      * Sets the default timeout used for user operations (using sessions and scanners).
      * Optional.
-     * If not provided, defaults to 10s.
+     * If not provided, defaults to 30s.
      * A value of 0 disables the timeout.
      * @param timeoutMs a timeout in milliseconds
      * @return this builder
@@ -2127,7 +2127,7 @@ public class AsyncKuduClient implements AutoCloseable {
     /**
      * Sets the default timeout to use when waiting on data from a socket.
      * Optional.
-     * If not provided, defaults to 5s.
+     * If not provided, defaults to 10s.
      * A value of 0 disables the timeout.
      * @param timeoutMs a timeout in milliseconds
      * @return this builder

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
index 6ca0939..0383a1c 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
@@ -289,7 +289,7 @@ public class KuduClient implements AutoCloseable {
      * Sets the default timeout used for administrative operations (e.g. createTable, deleteTable,
      * etc).
      * Optional.
-     * If not provided, defaults to 10s.
+     * If not provided, defaults to 30s.
      * A value of 0 disables the timeout.
      * @param timeoutMs a timeout in milliseconds
      * @return this builder
@@ -302,7 +302,7 @@ public class KuduClient implements AutoCloseable {
     /**
      * Sets the default timeout used for user operations (using sessions and scanners).
      * Optional.
-     * If not provided, defaults to 10s.
+     * If not provided, defaults to 30s.
      * A value of 0 disables the timeout.
      * @param timeoutMs a timeout in milliseconds
      * @return this builder
@@ -315,7 +315,7 @@ public class KuduClient implements AutoCloseable {
     /**
      * Sets the default timeout to use when waiting on data from a socket.
      * Optional.
-     * If not provided, defaults to 5s.
+     * If not provided, defaults to 10s.
      * A value of 0 disables the timeout.
      * @param timeoutMs a timeout in milliseconds
      * @return this builder

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/ITClient.java b/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
index 37d513e..c2b47e3 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
@@ -78,7 +78,6 @@ public class ITClient extends BaseKuduTest {
     // higher overall operation timeout.
     localAsyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
         .defaultSocketReadTimeoutMs(500)
-        .defaultOperationTimeoutMs(20000)
         .build();
     localClient = new KuduClient(localAsyncClient);
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/src/kudu/client/client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index e4d5321..ad5e14c 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -1347,7 +1347,7 @@ TEST_F(ClientTest, TestScanTimeout) {
     Status s = scanner.Open();
     EXPECT_TRUE(s.IsTimedOut()) << s.ToString();
     EXPECT_FALSE(scanner.data_->remote_) << "should not have located any tablet";
-    client_->data_->default_rpc_timeout_ = MonoDelta::FromSeconds(5);
+    client_->data_->default_rpc_timeout_ = MonoDelta::FromSeconds(10);
   }
 
   // Warm the cache so that the subsequent timeout occurs within the scan,
@@ -1365,7 +1365,6 @@ TEST_F(ClientTest, TestScanTimeout) {
 
   // Insert some more rows so that the scan takes multiple batches, instead of
   // fetching all the data on the 'Open()' call.
-  client_->data_->default_rpc_timeout_ = MonoDelta::FromSeconds(5);
   ASSERT_NO_FATAL_FAILURE(InsertTestRows(client_table_.get(), 1000, 1));
   {
     google::FlagSaver saver;
@@ -1377,7 +1376,6 @@ TEST_F(ClientTest, TestScanTimeout) {
     // scanner timeout instead.
     FLAGS_scanner_inject_latency_on_each_batch_ms = 50;
     client_->data_->default_rpc_timeout_ = MonoDelta::FromMilliseconds(1);
-    scanner.SetTimeoutMillis(5000);
 
     // Should successfully scan.
     ASSERT_OK(scanner.Open());

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/src/kudu/client/client.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index bf1d474..6ff0d65 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -136,12 +136,12 @@ class KUDU_EXPORT KuduClientBuilder {
   // The default timeout used for administrative operations (e.g. CreateTable,
   // AlterTable, ...). Optional.
   //
-  // If not provided, defaults to 10s.
+  // If not provided, defaults to 30s.
   KuduClientBuilder& default_admin_operation_timeout(const MonoDelta& timeout);
 
   // The default timeout for individual RPCs. Optional.
   //
-  // If not provided, defaults to 5s.
+  // If not provided, defaults to 10s.
   KuduClientBuilder& default_rpc_timeout(const MonoDelta& timeout);
 
   // Creates the client.
@@ -878,7 +878,7 @@ class KUDU_EXPORT KuduScanner {
 
   // Default scanner timeout.
   // This is set to 3x the default RPC timeout (see KuduClientBuilder::default_rpc_timeout()).
-  enum { kScanTimeoutMillis = 15000 };
+  enum { kScanTimeoutMillis = 30000 };
 
   // Initialize the scanner. The given 'table' object must remain valid
   // for the lifetime of this scanner object.

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/src/kudu/client/client_builder-internal.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client_builder-internal.cc b/src/kudu/client/client_builder-internal.cc
index 8e6dd03..cb56393 100644
--- a/src/kudu/client/client_builder-internal.cc
+++ b/src/kudu/client/client_builder-internal.cc
@@ -22,8 +22,8 @@ namespace kudu {
 namespace client {
 
 KuduClientBuilder::Data::Data()
-  : default_admin_operation_timeout_(MonoDelta::FromSeconds(10)),
-    default_rpc_timeout_(MonoDelta::FromSeconds(5)) {
+  : default_admin_operation_timeout_(MonoDelta::FromSeconds(30)),
+    default_rpc_timeout_(MonoDelta::FromSeconds(10)) {
 }
 
 KuduClientBuilder::Data::~Data() {

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/src/kudu/client/predicate-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/predicate-test.cc b/src/kudu/client/predicate-test.cc
index 76bf5d4..7bc4069 100644
--- a/src/kudu/client/predicate-test.cc
+++ b/src/kudu/client/predicate-test.cc
@@ -94,7 +94,6 @@ class PredicateTest : public KuduTest {
   int CountRows(const shared_ptr<KuduTable>& table,
                 const vector<KuduPredicate*>& predicates) {
     KuduScanner scanner(table.get());
-    CHECK_OK(scanner.SetTimeoutMillis(5000));
     for (KuduPredicate* predicate : predicates) {
       CHECK_OK(scanner.AddConjunctPredicate(predicate));
     }

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/01e8329c/src/kudu/integration-tests/all_types-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/all_types-itest.cc b/src/kudu/integration-tests/all_types-itest.cc
index 9dae94f..a304c47 100644
--- a/src/kudu/integration-tests/all_types-itest.cc
+++ b/src/kudu/integration-tests/all_types-itest.cc
@@ -376,7 +376,6 @@ class AllTypesItest : public KuduTest {
       RETURN_NOT_OK(scanner.SetBatchSizeBytes(KMaxBatchSize));
       RETURN_NOT_OK(scanner.SetFaultTolerant());
       RETURN_NOT_OK(scanner.SetReadMode(KuduScanner::READ_AT_SNAPSHOT));
-      RETURN_NOT_OK(scanner.SetTimeoutMillis(5000));
       RETURN_NOT_OK(scanner.Open());
       LOG(INFO) << "Scanning tablet: [" << low_split << ", " << high_split << ")";
 


[4/5] incubator-kudu git commit: [java client] Only add the Deferred to KuduRPC.toString if DEBUG is enabled

Posted by jd...@apache.org.
[java client] Only add the Deferred to KuduRPC.toString if DEBUG is enabled

All the KuduRPC daughter classes append their Deferred object when
toString() is called, which gives things like:

KuduRpc(method=Write, tablet=9d66b0bb6019492daca0a607fb0bd64b, attempt=8, DeadlineTracker(timeout=10000, elapsed=6289), Deferred@608776154(state=PENDING, result=null, callback=apply batch response -> callback: mark tablet 9d66b0bb6019492daca0a607fb0bd64b inflight done -> (continuation of Deferred@544649099 after retry RPC@1702110689) -> (continuation of Deferred@956406952 after retry RPC@1415267245) -> (continuation of Deferred@690509936 after retry RPC@756533645) -> (continuation of Deferred@1348564629 after retry RPC@1415510081), errback=apply batch error response -> errback: mark tablet 9d66b0bb6019492daca0a607fb0bd64b inflight done -> (continuation of Deferred@544649099 after retry RPC@1702110689) -> (continuation of Deferred@956406952 after retry RPC@1415267245) -> (continuation of Deferred@690509936 after retry RPC@756533645) -> (continuation of Deferred@1348564629 after retry RPC@1415510081)))

It can be useful to debug what happened to the chaining of deferreds
and callbacks, but usually it's not needed and it looks scary.

This patch makes it so that we only add the Deferred part if DEBUG
is enabled, so by default we'd instead output a more reasonable:

KuduRpc(method=Write, tablet=9d66b0bb6019492daca0a607fb0bd64b, attempt=8, DeadlineTracker(timeout=10000, elapsed=6289))

Change-Id: Iff08c234fef3ac0f0c6c8ec7c63078233c702f18
Reviewed-on: http://gerrit.cloudera.org:8080/2873
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/0eefc493
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/0eefc493
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/0eefc493

Branch: refs/heads/master
Commit: 0eefc4937583ee4d9a234cf0024d4c358e35689a
Parents: 01e8329
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Tue Apr 26 11:39:16 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Apr 28 02:31:38 2016 +0000

----------------------------------------------------------------------
 .../src/main/java/org/kududb/client/KuduRpc.java          | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/0eefc493/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java b/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
index 4219427..a3639e0 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
@@ -35,6 +35,8 @@ import org.jboss.netty.buffer.ChannelBuffers;
 import org.kududb.annotations.InterfaceAudience;
 import org.kududb.util.Pair;
 import org.kududb.util.Slice;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -63,6 +65,8 @@ public abstract class KuduRpc<R> {
   protected static final String MASTER_SERVICE_NAME = "kudu.master.MasterService";
   protected static final String TABLET_SERVER_SERVICE_NAME = "kudu.tserver.TabletServerService";
 
+  private static final Logger LOG = LoggerFactory.getLogger(KuduRpc.class);
+
   public interface HasKey {
     /**
      * Returns the partition key this RPC is for.
@@ -236,7 +240,11 @@ public abstract class KuduRpc<R> {
     }
     buf.append(", attempt=").append(attempt);
     buf.append(", ").append(deadlineTracker);
-    buf.append(", ").append(deferred);
+    // Cheating a bit, we're not actually logging but we'll augment the information provided by
+    // this method if DEBUG is enabled.
+    if (LOG.isDebugEnabled()) {
+      buf.append(", ").append(deferred);
+    }
     buf.append(')');
     return buf.toString();
   }


[5/5] incubator-kudu git commit: [java client] Add some missing annotations

Posted by jd...@apache.org.
[java client] Add some missing annotations

Change-Id: Iaa7e93256b958f238e0bc44c6a2eea6d6b0a1b06
Reviewed-on: http://gerrit.cloudera.org:8080/2880
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/c2aed1b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/c2aed1b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/c2aed1b7

Branch: refs/heads/master
Commit: c2aed1b7ca71819c768bc94280c9067872f755cb
Parents: 0eefc49
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Tue Apr 26 20:56:04 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Apr 28 02:32:03 2016 +0000

----------------------------------------------------------------------
 .../src/main/java/org/kududb/client/AsyncKuduClient.java           | 2 ++
 .../src/main/java/org/kududb/client/AsyncKuduScanner.java          | 2 ++
 java/kudu-client/src/main/java/org/kududb/client/KuduClient.java   | 2 ++
 3 files changed, 6 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c2aed1b7/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
index 5483288..fbe0170 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
@@ -2046,6 +2046,8 @@ public class AsyncKuduClient implements AutoCloseable {
    * Builder class to use in order to connect to Kudu.
    * All the parameters beyond those in the constructors are optional.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Evolving
   public final static class AsyncKuduClientBuilder {
     private static final int DEFAULT_MASTER_PORT = 7051;
     private static final int DEFAULT_BOSS_COUNT = 1;

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c2aed1b7/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
index 80c4f67..4365f15 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
@@ -814,6 +814,8 @@ public final class AsyncKuduScanner {
    * A Builder class to build {@link AsyncKuduScanner}.
    * Use {@link AsyncKuduClient#newScannerBuilder} in order to get a builder instance.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Evolving
   public static class AsyncKuduScannerBuilder
       extends AbstractKuduScannerBuilder<AsyncKuduScannerBuilder, AsyncKuduScanner> {
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c2aed1b7/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
index 0383a1c..a3a5529 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
@@ -254,6 +254,8 @@ public class KuduClient implements AutoCloseable {
    * Builder class to use in order to connect to Kudu.
    * All the parameters beyond those in the constructors are optional.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Evolving
   public final static class KuduClientBuilder {
     private AsyncKuduClient.AsyncKuduClientBuilder clientBuilder;
 


[2/5] incubator-kudu git commit: KUDU-1364. Don't clear the cache when a server disconnect

Posted by jd...@apache.org.
KUDU-1364. Don't clear the cache when a server disconnect

Folks seem to agree that, in the Java client, clearing the cache when a
server gets disconnected (for any reason) is not a good idea.

This patch does major surgery on how we handle cached TabletClients.
We now always keep them in RemoteTablet, and do our best to remember
if they were a leader. Only the ip2client is being removed from. We
now rely on the TabletClient's 'dead' attribute to know if a
connection is stale and needs to be re-established.

Not clear if client2tablets really needs to be removed from, and when.
We'll probably only remove from it in the future when we support
dropping ranges.

The patch also introduces a new integration test for the client. It's
still super basic but gives more confidence in the change.

It also fixes a dumb bug that was killing scanners too soon if there
was a disconnection.

Change-Id: I8606bfcedb09af57b66ba0f065067f0f3335a4a8
Reviewed-on: http://gerrit.cloudera.org:8080/2449
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/c4180bb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/c4180bb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/c4180bb0

Branch: refs/heads/master
Commit: c4180bb03bf3425756ea09966f004153a8a31f72
Parents: 3944122
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Thu Mar 3 20:58:38 2016 -0800
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Thu Apr 28 01:48:53 2016 +0000

----------------------------------------------------------------------
 .../java/org/kududb/client/AsyncKuduClient.java | 188 ++++++---
 .../org/kududb/client/AsyncKuduScanner.java     |   2 +
 .../java/org/kududb/client/TabletClient.java    |  51 ++-
 .../test/java/org/kududb/client/ITClient.java   | 396 +++++++++++++++++++
 .../org/kududb/client/TestAsyncKuduClient.java  |  20 +-
 5 files changed, 591 insertions(+), 66 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c4180bb0/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
index d7c5b4d..6482996 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
@@ -582,12 +582,12 @@ public class AsyncKuduClient implements AutoCloseable {
     final TabletClient client = clientFor(tablet);
     final KuduRpc<AsyncKuduScanner.Response> next_request = scanner.getNextRowsRequest();
     final Deferred<AsyncKuduScanner.Response> d = next_request.getDeferred();
-    if (client == null) {
-      // Oops, we no longer know anything about this client or tabletSlice.  Our
-      // cache was probably invalidated while the client was scanning.  This
-      // means that we lost the connection to that TabletServer, so we have to
-      // try to re-connect and check if the scanner is still good.
-      return sendRpcToTablet(next_request);
+    if (client == null || !client.isAlive()) {
+      // A null client means we either don't know about this tablet anymore (unlikely) or we
+      // couldn't find a leader (which could be triggered by a read timeout).
+      // We'll first delay the RPC in case things take some time to settle down, then retry.
+      delayedSendRpcToTablet(next_request, null);
+      return next_request.getDeferred();
     }
     next_request.attempt++;
     client.sendRpc(next_request);
@@ -596,8 +596,8 @@ public class AsyncKuduClient implements AutoCloseable {
 
   /**
    * Package-private access point for {@link AsyncKuduScanner}s to close themselves.
-   * @param scanner The scanner to close.
-   * @return A deferred object that indicates the completion of the request.
+   * @param scanner the scanner to close
+   * @return a deferred object that indicates the completion of the request.
    * The {@link AsyncKuduScanner.Response} can contain rows that were left to scan.
    */
   Deferred<AsyncKuduScanner.Response> closeScanner(final AsyncKuduScanner scanner) {
@@ -608,9 +608,9 @@ public class AsyncKuduClient implements AutoCloseable {
     }
 
     final TabletClient client = clientFor(tablet);
-    if (client == null) {
-      // Oops, we no longer know anything about this client or tabletSlice.  Our
-      // cache was probably invalidated while the client was scanning.  So
+    if (client == null || !client.isAlive()) {
+      // Oops, we couldn't find a tablet server that hosts this tablet. Our
+      // cache was probably invalidated while the client was scanning. So
       // we can't close this scanner properly.
       LOG.warn("Cannot close " + scanner + " properly, no connection open for "
           + (tablet == null ? null : tablet));
@@ -643,13 +643,45 @@ public class AsyncKuduClient implements AutoCloseable {
       request.setPropagatedTimestamp(lastPropagatedTs);
     }
 
+    // If we found a tablet, we'll try to find the TS to talk to. If that TS was previously
+    // disconnected, say because we didn't query that tablet for some seconds, then we'll try to
+    // reconnect based on the old information. If that fails, we'll instead continue with the next
+    // block that queries the master.
     if (tablet != null) {
       TabletClient tabletClient = clientFor(tablet);
       if (tabletClient != null) {
-        request.setTablet(tablet);
         final Deferred<R> d = request.getDeferred();
-        tabletClient.sendRpc(request);
-        return d;
+        if (tabletClient.isAlive()) {
+          request.setTablet(tablet);
+          tabletClient.sendRpc(request);
+          return d;
+        }
+        try {
+          tablet.reconnectTabletClient(tabletClient);
+        } catch (UnknownHostException e) {
+          LOG.error("Cached tablet server {}'s host cannot be resolved, will query the master",
+              tabletClient.getUuid(), e);
+          // Because of this exception, clientFor() below won't be able to find a newTabletClient
+          // and we'll delay the RPC.
+        }
+        TabletClient newTabletClient = clientFor(tablet);
+        assert (tabletClient != newTabletClient);
+
+        if (newTabletClient == null) {
+          // Wait a little bit before hitting the master.
+          delayedSendRpcToTablet(request, null);
+          return request.getDeferred();
+        }
+
+        if (!newTabletClient.isAlive()) {
+          LOG.debug("Tried reconnecting to tablet server {} but failed, " +
+              "will query the master", tabletClient.getUuid());
+          // Let fall through.
+        } else {
+          request.setTablet(tablet);
+          newTabletClient.sendRpc(request);
+          return d;
+        }
       }
     }
 
@@ -687,6 +719,7 @@ public class AsyncKuduClient implements AutoCloseable {
       this.request = request;
     }
     public Deferred<R> call(final D arg) {
+      LOG.debug("Retrying sending RPC {} after lookup", request);
       return sendRpcToTablet(request);  // Retry the RPC.
     }
     public String toString() {
@@ -937,6 +970,7 @@ public class AsyncKuduClient implements AutoCloseable {
     }
     final Exception e = new NonRecoverableException(message + request, cause);
     request.errback(e);
+    LOG.debug("Cannot continue with this RPC: {} because of: {}", request, message, e);
     return Deferred.fromError(e);
   }
 
@@ -1142,7 +1176,7 @@ public class AsyncKuduClient implements AutoCloseable {
   private void invalidateTabletCache(RemoteTablet tablet, TabletClient server) {
     LOG.info("Removing server " + server.getUuid() + " from this tablet's cache " +
         tablet.getTabletIdAsString());
-    tablet.removeTabletServer(server);
+    tablet.removeTabletClient(server);
   }
 
   /** Callback executed when a master lookup completes.  */
@@ -1218,7 +1252,7 @@ public class AsyncKuduClient implements AutoCloseable {
       // If we already know about this one, just refresh the locations
       RemoteTablet currentTablet = tablet2client.get(tabletId);
       if (currentTablet != null) {
-        currentTablet.refreshServers(tabletPb);
+        currentTablet.refreshTabletClients(tabletPb);
         continue;
       }
 
@@ -1231,7 +1265,7 @@ public class AsyncKuduClient implements AutoCloseable {
       }
       LOG.info("Discovered tablet {} for table {} with partition {}",
                tabletId.toString(Charset.defaultCharset()), tableName, rt.getPartition());
-      rt.refreshServers(tabletPb);
+      rt.refreshTabletClients(tabletPb);
       // This is making this tablet available
       // Even if two clients were racing in this method they are putting the same RemoteTablet
       // with the same start key in the CSLM in the end
@@ -1328,7 +1362,7 @@ public class AsyncKuduClient implements AutoCloseable {
         return client;
       }
       final TabletClientPipeline pipeline = new TabletClientPipeline();
-      client = pipeline.init(uuid);
+      client = pipeline.init(uuid, host, port);
       chan = channelFactory.newChannel(pipeline);
       ip2client.put(hostport, client);  // This is guaranteed to return null.
     }
@@ -1464,7 +1498,7 @@ public class AsyncKuduClient implements AutoCloseable {
           public ArrayList<Void> call(final ArrayList<Void> arg) {
             // Normally, now that we've shutdown() every client, all our caches should
             // be empty since each shutdown() generates a DISCONNECTED event, which
-            // causes TabletClientPipeline to call removeClientFromCache().
+            // causes TabletClientPipeline to call removeClientFromIpCache().
             HashMap<String, TabletClient> logme = null;
             synchronized (ip2client) {
               if (!ip2client.isEmpty()) {
@@ -1542,12 +1576,12 @@ public class AsyncKuduClient implements AutoCloseable {
   }
 
   /**
-   * Removes all the cache entries referred to the given client.
-   * @param client The client for which we must invalidate everything.
-   * @param remote The address of the remote peer, if known, or null.
+   * Removes the given client from the `ip2client` cache.
+   * @param client The client for which we must clear the ip cache
+   * @param remote The address of the remote peer, if known, or null
    */
-  private void removeClientFromCache(final TabletClient client,
-                                     final SocketAddress remote) {
+  private void removeClientFromIpCache(final TabletClient client,
+                                       final SocketAddress remote) {
 
     if (remote == null) {
       return;  // Can't continue without knowing the remote address.
@@ -1584,19 +1618,24 @@ public class AsyncKuduClient implements AutoCloseable {
           + hostport + "), it was found that there was no entry"
           + " corresponding to " + remote + ".  This shouldn't happen.");
     }
+  }
 
-    ArrayList<RemoteTablet> tablets = client2tablets.remove(client);
+  /**
+   * Call this method after encountering an error connecting to a tablet server so that we stop
+   * considering it a leader for the tablets it serves.
+   * @param client tablet server to use for demotion
+   */
+  void demoteAsLeaderForAllTablets(final TabletClient client) {
+    ArrayList<RemoteTablet> tablets = client2tablets.get(client);
     if (tablets != null) {
       // Make a copy so we don't need to synchronize on it while iterating.
       RemoteTablet[] tablets_copy;
       synchronized (tablets) {
         tablets_copy = tablets.toArray(new RemoteTablet[tablets.size()]);
-        tablets = null;
-        // If any other thread still has a reference to `tablets', their
-        // updates will be lost (and we don't care).
       }
       for (final RemoteTablet remoteTablet : tablets_copy) {
-        remoteTablet.removeTabletServer(client);
+        // It will be a no-op if it's not already a leader.
+        remoteTablet.demoteLeader(client);
       }
     }
   }
@@ -1620,8 +1659,8 @@ public class AsyncKuduClient implements AutoCloseable {
      */
     private boolean disconnected = false;
 
-    TabletClient init(String uuid) {
-      final TabletClient client = new TabletClient(AsyncKuduClient.this, uuid);
+    TabletClient init(String uuid, String host, int port) {
+      final TabletClient client = new TabletClient(AsyncKuduClient.this, uuid, host, port);
       if (defaultSocketReadTimeoutMs > 0) {
         super.addLast("timeout-handler",
             new ReadTimeoutHandler(timer,
@@ -1679,10 +1718,8 @@ public class AsyncKuduClient implements AutoCloseable {
           remote = slowSearchClientIP(client);
         }
 
-        // Prevent the client from buffering requests while we invalidate
-        // everything we have about it.
         synchronized (client) {
-          removeClientFromCache(client, remote);
+          removeClientFromIpCache(client, remote);
         }
       } catch (Exception e) {
         log.error("Uncaught exception when handling a disconnection of " + getChannel(), e);
@@ -1766,7 +1803,7 @@ public class AsyncKuduClient implements AutoCloseable {
    * succeeds.
    *
    * Subtleties:
-   * We don't keep track of a TS after it disconnects (via removeTabletServer), so if we
+   * We don't keep track of a TS after it disconnects (via removeTabletClient), so if we
    * haven't contacted one for 10 seconds (socket timeout), it will be removed from the list of
    * tabletServers. This means that if the leader fails, we only have one other TS to "promote"
    * or maybe none at all. This is partly why we then set leaderIndex to NO_LEADER_INDEX.
@@ -1793,7 +1830,7 @@ public class AsyncKuduClient implements AutoCloseable {
       this.partition = partition;
     }
 
-    void refreshServers(Master.TabletLocationsPB tabletLocations) throws NonRecoverableException {
+    void refreshTabletClients(Master.TabletLocationsPB tabletLocations) throws NonRecoverableException {
 
       synchronized (tabletServers) { // TODO not a fat lock with IP resolving in it
         tabletServers.clear();
@@ -1820,9 +1857,9 @@ public class AsyncKuduClient implements AutoCloseable {
             lookupExceptions.add(ex);
           }
         }
-        leaderIndex = 0;
+
         if (leaderIndex == NO_LEADER_INDEX) {
-          LOG.warn("No leader provided for tablet " + getTabletIdAsString());
+          LOG.warn("No leader provided for tablet {}", getTabletIdAsString());
         }
 
         // If we found a tablet that doesn't contain a single location that we can resolve, there's
@@ -1846,19 +1883,48 @@ public class AsyncKuduClient implements AutoCloseable {
 
       final ArrayList<RemoteTablet> tablets = client2tablets.get(client);
 
-      if (tablets == null) {
-        // We raced with removeClientFromCache and lost. The client we got was just disconnected.
-        // Reconnect.
-        addTabletClient(uuid, host, port, isLeader);
-      } else {
-        synchronized (tablets) {
-          if (isLeader) {
-            tabletServers.add(0, client);
-          } else {
-            tabletServers.add(client);
-          }
-          tablets.add(this);
+      synchronized (tablets) {
+        tabletServers.add(client);
+        if (isLeader) {
+          leaderIndex = tabletServers.size() - 1;
         }
+        tablets.add(this);
+      }
+    }
+
+    /**
+     * Call this method when an existing TabletClient in this tablet's cache is found to be dead.
+     * It removes the passed TS from this tablet's cache and replaces it with a new instance of
+     * TabletClient. It will keep its leader status if it was already considered a leader.
+     * If the passed TabletClient was already removed, then this is a no-op.
+     * @param staleTs TS to reconnect to
+     * @throws UnknownHostException if we can't resolve server's hostname
+     */
+    void reconnectTabletClient(TabletClient staleTs) throws UnknownHostException {
+      assert (!staleTs.isAlive());
+
+      synchronized (tabletServers) {
+        int index = tabletServers.indexOf(staleTs);
+
+        if (index == -1) {
+          // Another thread already took care of it.
+          return;
+        }
+
+        boolean wasLeader = index == leaderIndex;
+
+        LOG.debug("Reconnecting to server {} for tablet {}. Was a leader? {}",
+            staleTs.getUuid(), getTabletIdAsString(), wasLeader);
+
+        boolean removed = removeTabletClient(staleTs);
+
+        if (!removed) {
+          LOG.debug("{} was already removed from tablet {}'s cache when reconnecting to it",
+              staleTs.getUuid(), getTabletIdAsString());
+        }
+
+        addTabletClient(staleTs.getUuid(), staleTs.getHost(),
+            staleTs.getPort(), wasLeader);
       }
     }
 
@@ -1873,7 +1939,7 @@ public class AsyncKuduClient implements AutoCloseable {
      * @param ts A TabletClient that was disconnected.
      * @return True if this method removed ts from the list, else false.
      */
-    boolean removeTabletServer(TabletClient ts) {
+    boolean removeTabletClient(TabletClient ts) {
       synchronized (tabletServers) {
         // TODO unit test for this once we have the infra
         int index = tabletServers.indexOf(ts);
@@ -1894,10 +1960,10 @@ public class AsyncKuduClient implements AutoCloseable {
     }
 
     /**
-     * If the passed TabletClient is the current leader, then the next one in the list will be
-     * "promoted" unless we're at the end of the list, in which case we set the leaderIndex to
-     * NO_LEADER_INDEX which will force a call to the master.
-     * @param ts A TabletClient that gave a sign that it isn't this tablet's leader.
+     * Clears the leader index if the passed tablet server is the current leader.
+     * If it is the current leader, then the next call to this tablet will have
+     * to query the master to find the new leader.
+     * @param ts a TabletClient that gave a sign that it isn't this tablet's leader
      */
     void demoteLeader(TabletClient ts) {
       synchronized (tabletServers) {
@@ -1905,15 +1971,17 @@ public class AsyncKuduClient implements AutoCloseable {
         // If this TS was removed or we're already forcing a call to the master (meaning someone
         // else beat us to it), then we just noop.
         if (index == -1 || leaderIndex == NO_LEADER_INDEX) {
+          LOG.debug("{} couldn't be demoted as the leader for {}",
+              ts.getUuid(), getTabletIdAsString());
           return;
         }
 
         if (leaderIndex == index) {
-          if (leaderIndex + 1 == tabletServers.size()) {
-            leaderIndex = NO_LEADER_INDEX;
-          } else {
-            leaderIndex++;
-          }
+          leaderIndex = NO_LEADER_INDEX;
+          LOG.debug("{} was demoted as the leader for {}", ts.getUuid(), getTabletIdAsString());
+        } else {
+          LOG.debug("{} wasn't the leader for {}, current leader is at index {}", ts.getUuid(),
+              getTabletIdAsString(), leaderIndex);
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c4180bb0/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
index cabca8d..80c4f67 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
@@ -743,11 +743,13 @@ public final class AsyncKuduScanner {
                  .setBatchSizeBytes(batchSizeBytes);
           break;
         case NEXT:
+          setTablet(AsyncKuduScanner.this.tablet);
           builder.setScannerId(ZeroCopyLiteralByteString.wrap(scannerId))
                  .setCallSeqId(sequenceId)
                  .setBatchSizeBytes(batchSizeBytes);
           break;
         case CLOSING:
+          setTablet(AsyncKuduScanner.this.tablet);
           builder.setScannerId(ZeroCopyLiteralByteString.wrap(scannerId))
                  .setBatchSizeBytes(0)
                  .setCloseScanner(true);

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c4180bb0/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java b/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
index e18eb55..3f91ece 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
@@ -26,8 +26,7 @@
 
 package org.kududb.client;
 
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
+import com.google.common.annotations.VisibleForTesting;
 import com.stumbleupon.async.Deferred;
 
 import org.jboss.netty.handler.timeout.ReadTimeoutException;
@@ -136,14 +135,20 @@ public class TabletClient extends ReplayingDecoder<VoidEnum> {
 
   private final String uuid;
 
+  private final String host;
+
+  private final int port;
+
   private final long socketReadTimeoutMs;
 
   private SecureRpcHelper secureRpcHelper;
 
-  public TabletClient(AsyncKuduClient client, String uuid) {
+  public TabletClient(AsyncKuduClient client, String uuid, String host, int port) {
     this.kuduClient = client;
     this.uuid = uuid;
     this.socketReadTimeoutMs = client.getDefaultSocketReadTimeoutMs();
+    this.host = host;
+    this.port = port;
   }
 
   <R> void sendRpc(KuduRpc<R> rpc) {
@@ -252,6 +257,22 @@ public class TabletClient extends ReplayingDecoder<VoidEnum> {
     return payload;
   }
 
+  /**
+   * Quick and dirty way to close a connection to a tablet server, if it wasn't already closed.
+   */
+  @VisibleForTesting
+  void disconnect() {
+    Channel chancopy = chan;
+    if (chancopy != null && chancopy.isConnected()) {
+      Channels.disconnect(chancopy);
+    }
+  }
+
+  /**
+   * Forcefully shuts down the connection to this tablet server and fails all the outstanding RPCs.
+   * Only use when shutting down a client.
+   * @return deferred object to use to track the shutting down of this connection
+   */
   public Deferred<Void> shutdown() {
     // First, check whether we have RPCs in flight and cancel them.
     for (Iterator<KuduRpc<?>> ite = rpcs_inflight.values().iterator(); ite
@@ -667,10 +688,12 @@ public class TabletClient extends ReplayingDecoder<VoidEnum> {
   private void failOrRetryRpc(final KuduRpc<?> rpc,
                               final ConnectionResetException exception) {
     AsyncKuduClient.RemoteTablet tablet = rpc.getTablet();
+    // Note As of the time of writing (03/11/16), a null tablet doesn't make sense, if we see a null
+    // tablet it's because we didn't set it properly before calling sendRpc().
     if (tablet == null) {  // Can't retry, dunno where this RPC should go.
       rpc.errback(exception);
     } else {
-      kuduClient.handleTabletNotFound(rpc, exception, this);
+      kuduClient.handleRetryableError(rpc, exception);
     }
   }
 
@@ -691,6 +714,9 @@ public class TabletClient extends ReplayingDecoder<VoidEnum> {
       cleanup(c);
     } else {
       LOG.error(getPeerUuidLoggingString() + "Unexpected exception from downstream on " + c, e);
+      // For any other exception, likely a connection error, we clear the leader state
+      // for those tablets that this TS is the cached leader of.
+      kuduClient.demoteAsLeaderForAllTablets(this);
     }
     if (c.isOpen()) {
       Channels.close(c);  // Will trigger channelClosed(), which will cleanup()
@@ -757,6 +783,23 @@ public class TabletClient extends ReplayingDecoder<VoidEnum> {
     return uuid;
   }
 
+  /**
+   * Returns this tablet server's port.
+   * @return a port number that this tablet server is bound to
+   */
+  int getPort() {
+    return port;
+  }
+
+  /**
+   * Returns this tablet server's hostname. We might get many hostnames from the master for a single
+   * TS, and this is the one we picked to connect to originally.
+   * @returna string that contains this tablet server's hostname
+   */
+  String getHost() {
+    return host;
+  }
+
   public String toString() {
     final StringBuilder buf = new StringBuilder(13 + 10 + 6 + 64 + 7 + 32 + 16 + 1 + 17 + 2 + 1);
     buf.append("TabletClient@")           // =13

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c4180bb0/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/ITClient.java b/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
new file mode 100644
index 0000000..37d513e
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
@@ -0,0 +1,396 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Integration test for the client. RPCs are sent to Kudu from multiple threads while processes
+ * are restarted and failures are injected.
+ *
+ * By default this test runs for 60 seconds, but this can be changed by passing a different value
+ * in "itclient.runtime.seconds". For example:
+ * "mvn test -Dtest=ITClient -Ditclient.runtime.seconds=120".
+ */
+public class ITClient extends BaseKuduTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ITClient.class);
+
+  private static final String RUNTIME_PROPERTY_NAME = "itclient.runtime.seconds";
+  private static final long DEFAULT_RUNTIME_SECONDS = 60;
+  // Time we'll spend waiting at the end of the test for things to settle. Also the minimum this
+  // test can run for.
+  private static final long TEST_MIN_RUNTIME_SECONDS = 2;
+  private static final long TEST_TIMEOUT_SECONDS = 600000;
+
+  private static final String TABLE_NAME =
+      ITClient.class.getName() + "-" + System.currentTimeMillis();
+  // One error and we stop the test.
+  private static final CountDownLatch KEEP_RUNNING_LATCH = new CountDownLatch(1);
+  // Latch used to track if an error occurred and we need to stop the test early.
+  private static final CountDownLatch ERROR_LATCH = new CountDownLatch(1);
+
+  private static KuduClient localClient;
+  private static AsyncKuduClient localAsyncClient;
+  private static KuduTable table;
+  private static long runtimeInSeconds;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+
+    String runtimeProp = System.getProperty(RUNTIME_PROPERTY_NAME);
+    runtimeInSeconds = runtimeProp == null ? DEFAULT_RUNTIME_SECONDS : Long.parseLong(runtimeProp);
+
+    if (runtimeInSeconds < TEST_MIN_RUNTIME_SECONDS || runtimeInSeconds > TEST_TIMEOUT_SECONDS) {
+      Assert.fail("This test needs to run more more than " + TEST_MIN_RUNTIME_SECONDS + " seconds" +
+          " and less than " + TEST_TIMEOUT_SECONDS + " seconds");
+    }
+
+    LOG.info ("Test running for {} seconds", runtimeInSeconds);
+
+    BaseKuduTest.setUpBeforeClass();
+
+    // Client we're using has low tolerance for read timeouts but a
+    // higher overall operation timeout.
+    localAsyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
+        .defaultSocketReadTimeoutMs(500)
+        .defaultOperationTimeoutMs(20000)
+        .build();
+    localClient = new KuduClient(localAsyncClient);
+
+    CreateTableOptions builder = new CreateTableOptions().setNumReplicas(3);
+    table = localClient.createTable(TABLE_NAME, basicSchema, builder);
+  }
+
+  @Test(timeout = TEST_TIMEOUT_SECONDS)
+  public void test() throws Exception {
+    ArrayList<Thread> threads = new ArrayList<>();
+    Thread chaosThread = new Thread(new ChaosThread());
+    Thread writerThread = new Thread(new WriterThread());
+    Thread scannerThread = new Thread(new ScannerThread());
+
+    threads.add(chaosThread);
+    threads.add(writerThread);
+    threads.add(scannerThread);
+
+    for (Thread thread : threads) {
+      thread.start();
+    }
+
+    // await() returns yes if the latch reaches 0, we don't want that.
+    Assert.assertFalse("Look for the last ERROR line in the log that comes from ITCLient",
+        ERROR_LATCH.await(runtimeInSeconds, TimeUnit.SECONDS));
+
+    // Indicate we want to stop, then wait a little bit for it to happen.
+    KEEP_RUNNING_LATCH.countDown();
+
+    for (Thread thread : threads) {
+      thread.interrupt();
+      thread.join();
+    }
+
+    AsyncKuduScanner scannerBuilder = localAsyncClient.newScannerBuilder(table).build();
+    int rowCount = countRowsInScan(scannerBuilder);
+    Assert.assertTrue(rowCount + " should be higher than 0", rowCount > 0);
+  }
+
+  /**
+   * Logs an error message and triggers the error count down latch, stopping this test.
+   * @param message error message to print
+   * @param exception optional exception to print
+   */
+  private void reportError(String message, Exception exception) {
+    LOG.error(message, exception);
+    ERROR_LATCH.countDown();
+  }
+
+  /**
+   * Thread that introduces chaos in the cluster, one at a time.
+   */
+  class ChaosThread implements Runnable {
+
+    private final Random random = new Random();
+
+    @Override
+    public void run() {
+      try {
+        KEEP_RUNNING_LATCH.await(2, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        return;
+      }
+      while (KEEP_RUNNING_LATCH.getCount() > 0) {
+        try {
+          boolean shouldContinue;
+          if (System.currentTimeMillis() % 2 == 0) {
+            shouldContinue = restartTS();
+          } else {
+
+            shouldContinue = disconnectNode();
+          }
+          // TODO restarting the master currently finds more bugs. Also, adding it to the list makes
+          // it necessary to find a new weighing mechanism betweent he different chaos options.
+          // shouldContinue = restartMaster();
+
+          if (!shouldContinue) {
+            return;
+          }
+          KEEP_RUNNING_LATCH.await(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+          return;
+        }
+
+      }
+    }
+
+    /**
+     * Failure injection. Picks a random tablet server from the client's cache and force
+     * disconects it.
+     * @return true if successfully completed or didn't find a server to disconnect, false it it
+     * encountered a failure
+     */
+    private boolean disconnectNode() {
+      try {
+        if (localAsyncClient.getTableClients().size() == 0) {
+          return true;
+        }
+
+        int tsToDisconnect = random.nextInt(localAsyncClient.getTableClients().size());
+        localAsyncClient.getTableClients().get(tsToDisconnect).disconnect();
+
+      } catch (Exception e) {
+        if (KEEP_RUNNING_LATCH.getCount() == 0) {
+          // Likely shutdown() related.
+          return false;
+        }
+        reportError("Couldn't disconnect a TS", e);
+        return false;
+      }
+      return true;
+    }
+
+    /**
+     * Forces the restart of a random tablet server.
+     * @return true if it successfully completed, false if it failed
+     */
+    private boolean restartTS() {
+      try {
+        BaseKuduTest.restartTabletServer(table);
+      } catch (Exception e) {
+        reportError("Couldn't restart a TS", e);
+        return false;
+      }
+      return true;
+    }
+
+    /**
+     * Forces the restart of the master.
+     * @return true if it successfully completed, false if it failed
+     */
+    private boolean restartMaster() {
+      try {
+        BaseKuduTest.restartLeaderMaster();
+      } catch (Exception e) {
+        reportError("Couldn't restart a master", e);
+        return false;
+      }
+      return true;
+    }
+
+  }
+
+  /**
+   * Thread that writes sequentially to the table. Every 10 rows it considers setting the flush mode
+   * to MANUAL_FLUSH or AUTO_FLUSH_SYNC.
+   */
+  class WriterThread implements Runnable {
+
+    private final KuduSession session = localClient.newSession();
+    private final Random random = new Random();
+    private int currentRowKey = 0;
+
+    @Override
+    public void run() {
+      session.setIgnoreAllDuplicateRows(true);
+      while (KEEP_RUNNING_LATCH.getCount() > 0) {
+        try {
+          OperationResponse resp = session.apply(createBasicSchemaInsert(table, currentRowKey));
+          if (hasRowErrorAndReport(resp)) {
+            return;
+          }
+          currentRowKey++;
+
+          // Every 10 rows we flush and change the flush mode randomly.
+          if (currentRowKey % 10 == 0) {
+
+            // First flush any accumulated rows before switching.
+            List<OperationResponse> responses = session.flush();
+            if (responses != null) {
+              for (OperationResponse batchedResp : responses) {
+                if (hasRowErrorAndReport(batchedResp)) {
+                  return;
+                }
+              }
+            }
+
+            if (random.nextBoolean()) {
+              session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+            } else {
+              session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
+            }
+          }
+        } catch (Exception e) {
+          if (KEEP_RUNNING_LATCH.getCount() == 0) {
+            // Likely shutdown() related.
+            return;
+          }
+          reportError("Got error while inserting row " + currentRowKey, e);
+          return;
+        }
+      }
+    }
+
+    private boolean hasRowErrorAndReport(OperationResponse resp) {
+      if (resp != null && resp.hasRowError()) {
+        reportError("The following RPC " + resp.getOperation().getRow() +
+            " returned this error: " + resp.getRowError(), null);
+        return true;
+      }
+      return false;
+    }
+  }
+
+  /**
+   * Thread that scans the table. Alternates randomly between random gets and full table scans.
+   */
+  class ScannerThread implements Runnable {
+
+    private final Random random = new Random();
+
+    // Updated by calling a full scan.
+    private int lastRowCount = 0;
+
+    @Override
+    public void run() {
+      while (KEEP_RUNNING_LATCH.getCount() > 0) {
+
+        boolean shouldContinue;
+
+        // Always scan until we find rows.
+        if (lastRowCount == 0 || random.nextBoolean()) {
+          shouldContinue = fullScan();
+        } else {
+          shouldContinue = randomGet();
+        }
+
+        if (!shouldContinue) {
+          return;
+        }
+
+        if (lastRowCount == 0) {
+          try {
+            KEEP_RUNNING_LATCH.await(50, TimeUnit.MILLISECONDS);
+          } catch (InterruptedException e) {
+            // Test is stopping.
+            return;
+          }
+        }
+      }
+    }
+
+    /**
+     * Reads a row at random that it knows to exist (smaller than lastRowCount).
+     * @return
+     */
+    private boolean randomGet() {
+      int key = random.nextInt(lastRowCount);
+      KuduPredicate predicate = KuduPredicate.newComparisonPredicate(
+          table.getSchema().getColumnByIndex(0), KuduPredicate.ComparisonOp.EQUAL, key);
+      KuduScanner scanner = localClient.newScannerBuilder(table).addPredicate(predicate).build();
+
+      List<RowResult> results = new ArrayList<>();
+      while (scanner.hasMoreRows()) {
+        try {
+          RowResultIterator ite = scanner.nextRows();
+          for (RowResult row : ite) {
+            results.add(row);
+          }
+        } catch (Exception e) {
+          return checkAndReportError("Got error while getting row " + key, e);
+        }
+      }
+
+      if (results.isEmpty() || results.size() > 1) {
+        reportError("Random get got 0 or many rows " + results.size() + " for key " + key, null);
+        return false;
+      }
+
+      int receivedKey = results.get(0).getInt(0);
+      if (receivedKey != key) {
+        reportError("Tried to get key " + key + " and received " + receivedKey, null);
+        return false;
+      }
+      return true;
+    }
+
+    /**
+     * Rusn a full table scan and updates the lastRowCount.
+     * @return
+     */
+    private boolean fullScan() {
+      AsyncKuduScanner scannerBuilder = localAsyncClient.newScannerBuilder(table).build();
+      try {
+        int rowCount = countRowsInScan(scannerBuilder);
+        if (rowCount < lastRowCount) {
+          reportError("Row count regressed: " + rowCount + " < " + lastRowCount, null);
+          return false;
+        }
+        lastRowCount = rowCount;
+        LOG.info("New row count {}", lastRowCount);
+      } catch (Exception e) {
+        checkAndReportError("Got error while row counting", e);
+      }
+      return true;
+    }
+
+    /**
+     * Checks the passed exception contains "Scanner not found". If it does then it returns true,
+     * else it reports the error and returns false.
+     * We need to do this because the scans in this client aren't fault tolerant.
+     * @param message message to print if the exception contains a real error
+     * @param e the exception to check
+     * @return true if the scanner failed because it wasn't false, otherwise false
+     */
+    private boolean checkAndReportError(String message, Exception e) {
+      if (!e.getCause().getMessage().contains("Scanner not found")) {
+        reportError(message, e);
+        return false;
+      }
+      return true;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c4180bb0/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
index 7b4c932..ff0097b 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
@@ -17,6 +17,7 @@
 package org.kududb.client;
 
 import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
 import com.google.protobuf.ByteString;
 import com.stumbleupon.async.Deferred;
 import org.junit.BeforeClass;
@@ -48,7 +49,7 @@ public class TestAsyncKuduClient extends BaseKuduTest {
     assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
 
     // 2. Disconnect the TabletClient.
-    client.getTableClients().get(0).shutdown().join(DEFAULT_SLEEP);
+    disconnectAndWait();
 
     // 3. Count again, it will trigger a re-connection and we should not hang or fail to scan.
     assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
@@ -76,11 +77,26 @@ public class TestAsyncKuduClient extends BaseKuduTest {
         rowCount, numRows);
 
     // 4. Disconnect the TS.
-    client.getTableClients().get(0).shutdown().join(DEFAULT_SLEEP);
+    disconnectAndWait();
+
     // 5. Make sure that we can continue scanning and that we get the remaining rows back.
     assertEquals(rowCount - numRows, countRowsInScan(scanner));
   }
 
+  private void disconnectAndWait() throws InterruptedException {
+    client.getTableClients().get(0).disconnect();
+    Stopwatch sw = new Stopwatch().start();
+    while (sw.elapsedMillis() < DEFAULT_SLEEP) {
+      if (!client.getTableClients().isEmpty()) {
+        Thread.sleep(50);
+        continue;
+      } else {
+        break;
+      }
+    }
+    assertTrue(client.getTableClients().isEmpty());
+  }
+
   @Test
   public void testBadHostnames() throws Exception {
     String badHostname = "some-unknown-host-hopefully";