You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/01/26 02:30:34 UTC
[1/2] incubator-kudu git commit: [java] waitForTabletServers should
belong to the mini cluster
Repository: incubator-kudu
Updated Branches:
refs/heads/master ffd84cf9c -> 7900bd5ce
[java] waitForTabletServers should belong to the mini cluster
Waiting for the number of tservers to reach a certain point is a
common enough need that it should be part of the mini cluster.
Change-Id: I527b09e3dc530f186a7f5cf141c3f460613cf51c
Reviewed-on: http://gerrit.cloudera.org:8080/1875
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/76e285e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/76e285e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/76e285e7
Branch: refs/heads/master
Commit: 76e285e7f267da5b14f87cebf12cc28b4b28bd3b
Parents: ffd84cf
Author: Jean-Daniel Cryans <jd...@cloudera.com>
Authored: Fri Jan 22 09:26:45 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Jan 25 19:58:10 2016 +0000
----------------------------------------------------------------------
.../java/org/kududb/client/BaseKuduTest.java | 20 +--------
.../java/org/kududb/client/MiniKuduCluster.java | 45 ++++++++++++++++++--
2 files changed, 44 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/76e285e7/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
index d1a1410..c6f06b5 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
@@ -77,6 +77,7 @@ public class BaseKuduTest {
miniCluster = new MiniKuduCluster.MiniKuduClusterBuilder()
.numMasters(NUM_MASTERS)
.numTservers(NUM_TABLET_SERVERS)
+ .defaultTimeoutMs(DEFAULT_SLEEP)
.build();
masterAddresses = miniCluster.getMasterAddresses();
masterHostPorts = miniCluster.getMasterHostPorts();
@@ -85,28 +86,11 @@ public class BaseKuduTest {
client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses).build();
syncClient = new KuduClient(client);
LOG.info("Waiting for tablet servers...");
- if (!waitForTabletServers(NUM_TABLET_SERVERS)) {
+ if (!miniCluster.waitForTabletServers(NUM_TABLET_SERVERS)) {
fail("Couldn't get " + NUM_TABLET_SERVERS + " tablet servers running, aborting");
}
}
- /**
- * Wait up to DEFAULT_SLEEP for an expected count of TS to connect to the master
- * @param expected How many TS are expected
- * @return true if there are at least as many TS as expected, otherwise false
- */
- static boolean waitForTabletServers(int expected) throws Exception {
- int count = 0;
- Stopwatch stopwatch = new Stopwatch().start();
- while (count < expected && stopwatch.elapsedMillis() < DEFAULT_SLEEP) {
- Thread.sleep(200);
- Deferred<ListTabletServersResponse> d = client.listTabletServers();
- d.addErrback(defaultErrorCB);
- count = d.join(DEFAULT_SLEEP).getTabletServersCount();
- }
- return count >= expected;
- }
-
@AfterClass
public static void tearDownAfterClass() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/76e285e7/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
index 4ec0f60..9bd5442 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
@@ -15,6 +15,7 @@ package org.kududb.client;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.net.HostAndPort;
import org.apache.commons.io.FileUtils;
@@ -28,7 +29,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
-import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Iterator;
@@ -62,10 +62,37 @@ public class MiniKuduCluster implements AutoCloseable {
private final List<String> pathsToDelete = new ArrayList<>();
private final List<HostAndPort> masterHostPorts = new ArrayList<>();
+ // Client we can use for common operations.
+ private final KuduClient syncClient;
+ private final int defaultTimeoutMs;
+
private String masterAddresses;
- private MiniKuduCluster(int numMasters, int numTservers) throws Exception {
+ private MiniKuduCluster(int numMasters, int numTservers, int defaultTimeoutMs) throws Exception {
+ this.defaultTimeoutMs = defaultTimeoutMs;
+
startCluster(numMasters, numTservers);
+
+ syncClient = new KuduClient.KuduClientBuilder(getMasterAddresses())
+ .defaultAdminOperationTimeoutMs(defaultTimeoutMs)
+ .defaultOperationTimeoutMs(defaultTimeoutMs)
+ .build();
+ }
+
+ /**
+ * Wait up to this instance's "default timeout" for an expected count of TS to
+ * connect to the master.
+ * @param expected How many TS are expected
+ * @return true if there are at least as many TS as expected, otherwise false
+ */
+ public boolean waitForTabletServers(int expected) throws Exception {
+ int count = 0;
+ Stopwatch stopwatch = new Stopwatch().start();
+ while (count < expected && stopwatch.elapsedMillis() < defaultTimeoutMs) {
+ Thread.sleep(200);
+ count = syncClient.listTabletServers().getTabletServersCount();
+ }
+ return count >= expected;
}
/**
@@ -369,6 +396,7 @@ public class MiniKuduCluster implements AutoCloseable {
private int numMasters = 1;
private int numTservers = 3;
+ private int defaultTimeoutMs = 50000;
public MiniKuduClusterBuilder numMasters(int numMasters) {
this.numMasters = numMasters;
@@ -380,8 +408,19 @@ public class MiniKuduCluster implements AutoCloseable {
return this;
}
+ /**
+ * Configures the internal client to use the given timeout for all operations. Also uses the
+ * timeout for tasks like waiting for tablet servers to check in with the master.
+ * @param defaultTimeoutMs timeout in milliseconds
+ * @return this instance
+ */
+ public MiniKuduClusterBuilder defaultTimeoutMs(int defaultTimeoutMs) {
+ this.defaultTimeoutMs = defaultTimeoutMs;
+ return this;
+ }
+
public MiniKuduCluster build() throws Exception {
- return new MiniKuduCluster(numMasters, numTservers);
+ return new MiniKuduCluster(numMasters, numTservers, defaultTimeoutMs);
}
}
[2/2] incubator-kudu git commit: client: fix missing includes in
scan_batch.h
Posted by to...@apache.org.
client: fix missing includes in scan_batch.h
scan_batch.h was missing some includes for gutil macros that it used.
This didn't cause problems in our own build, because we typically
had included the right files above including this header. But this
broke an Impala build where the caller wasn't explicitly including
other code ahead of this.
This adds a new bit of code to client_samples-test which verifies
that the installed library's headers are all buildable on their own.
This isn't _exactly_ part of the job description of the 'samples' test,
but we already had the build and install code there and didn't want
to duplicate it, nor did I want to take the time to refactor a bunch
of shell code at this juncture.
Change-Id: I8e10f406c80dff7cbc61ca609d8d2bde737b91b6
Reviewed-on: http://gerrit.cloudera.org:8080/1896
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/7900bd5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/7900bd5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/7900bd5c
Branch: refs/heads/master
Commit: 7900bd5ce73a203aff565aae0996a8e53c4926ea
Parents: 76e285e
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Jan 25 14:45:14 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Jan 26 01:28:58 2016 +0000
----------------------------------------------------------------------
src/kudu/client/client_samples-test.sh | 22 ++++++++++++++++++----
src/kudu/client/scan_batch.h | 7 +++++++
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7900bd5c/src/kudu/client/client_samples-test.sh
----------------------------------------------------------------------
diff --git a/src/kudu/client/client_samples-test.sh b/src/kudu/client/client_samples-test.sh
index 7925261..f573207 100755
--- a/src/kudu/client/client_samples-test.sh
+++ b/src/kudu/client/client_samples-test.sh
@@ -17,10 +17,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Tests that the Kudu client sample code can be built out-of-tree and runs
-# properly.
-
-set -e
+# Tests that the Kudu client library can be installed outside
+# the build tree, that the installed headers are sane, and that
+# the sample code can be built and runs correctly.
# Clean up after the test. Must be idempotent.
cleanup() {
@@ -57,6 +56,21 @@ else
fi
popd
+# Test that all of the installed headers can be compiled on their own.
+# This catches bugs where we've made a mistake in 'include-what-you-use'
+# within the library.
+for include_file in $(find $LIBRARY_DIR -name \*.h) ; do
+ echo Checking standalone compilation of $include_file...
+ if ! ${CXX:-g++} -o /dev/null -I$LIBRARY_DIR/usr/local/include $include_file ; then
+ set +x
+ echo
+ echo -----------------------------------------
+ echo $include_file fails to build on its own.
+ echo See log above for details.
+ echo -----------------------------------------
+ exit 1
+ fi
+done
# Prefer the cmake on the system path, since we expect our client library
# to be usable with older versions of cmake. But if it isn't there,
# use the one from thirdparty.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7900bd5c/src/kudu/client/scan_batch.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/scan_batch.h b/src/kudu/client/scan_batch.h
index c08130a..3d510d4 100644
--- a/src/kudu/client/scan_batch.h
+++ b/src/kudu/client/scan_batch.h
@@ -19,6 +19,13 @@
#include <string>
+#ifdef KUDU_HEADERS_NO_STUBS
+#include "kudu/gutil/macros.h"
+#include "kudu/gutil/port.h"
+#else
+#include "kudu/client/stubs.h"
+#endif
+
#include "kudu/util/kudu_export.h"
#include "kudu/util/slice.h"