You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2017/07/05 19:34:08 UTC
kudu git commit: [java] add BaseKuduTest.doSetup() method
Repository: kudu
Updated Branches:
refs/heads/master 1070e764d -> 7e7452704
[java] add BaseKuduTest.doSetup() method
Added BaseKuduTest.doSetup() to call from custom @BeforeClass methods.
This is to allow for run-time customization of MiniKuduCluster
parameters such as number of masters, number of tablet servers and
default timeout for various operations. The new method is needed for
a follow-up commit.
Also did minor clean-up on some of the related files affected by the
change above.
Change-Id: Id283bcb6a6d109839d86932cb807a1da8156f6eb
Reviewed-on: http://gerrit.cloudera.org:8080/7336
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7e745270
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7e745270
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7e745270
Branch: refs/heads/master
Commit: 7e7452704bb5295196f04423c86d0bc317c25311
Parents: 1070e76
Author: Alexey Serbin <as...@cloudera.com>
Authored: Thu Jun 29 13:55:15 2017 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Wed Jul 5 19:32:07 2017 +0000
----------------------------------------------------------------------
.../org/apache/kudu/client/AsyncKuduClient.java | 4 +-
.../apache/kudu/client/AsyncKuduScanner.java | 4 +-
.../org/apache/kudu/client/BaseKuduTest.java | 78 +++++++++++---------
.../java/org/apache/kudu/client/ITClient.java | 3 +-
.../org/apache/kudu/client/MiniKuduCluster.java | 4 +-
.../apache/kudu/client/TestAsyncKuduClient.java | 6 +-
.../kudu/client/TestAsyncKuduSession.java | 3 +-
.../org/apache/kudu/client/TestKuduClient.java | 4 +-
.../org/apache/kudu/client/TestKuduSession.java | 2 +-
.../org/apache/kudu/client/TestNegotiator.java | 1 -
.../org/apache/kudu/flume/sink/KuduSink.java | 1 -
.../apache/kudu/flume/sink/KuduSinkTest.java | 1 -
.../apache/kudu/mapreduce/ITInputFormatJob.java | 1 -
13 files changed, 52 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index 3a83c59..019a3f5 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -126,7 +126,7 @@ public class AsyncKuduClient implements AutoCloseable {
* a lookup of a single partition (e.g. for a write), or re-looking-up a tablet with
* stale information.
*/
- static int FETCH_TABLETS_PER_POINT_LOOKUP = 10;
+ static final int FETCH_TABLETS_PER_POINT_LOOKUP = 10;
/**
* The number of tablets to fetch from the master when looking up a range of
* tablets.
@@ -680,7 +680,7 @@ public class AsyncKuduClient implements AutoCloseable {
void removeSession(AsyncKuduSession session) {
synchronized (sessions) {
boolean removed = sessions.remove(session);
- assert removed == true;
+ assert removed;
}
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
index 8d42de7..5755158 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
@@ -555,9 +555,7 @@ public final class AsyncKuduScanner {
if (closed) {
return Deferred.fromResult(null);
}
- final Deferred<RowResultIterator> d =
- client.closeScanner(this).addCallback(closedCallback()); // TODO errBack ?
- return d;
+ return client.closeScanner(this).addCallback(closedCallback()); // TODO errBack ?
}
/** Callback+Errback invoked when the TabletServer closed our scanner. */
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
index 190ba06..379fadf 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
@@ -45,15 +45,8 @@ public class BaseKuduTest {
protected static final Logger LOG = LoggerFactory.getLogger(BaseKuduTest.class);
- private static final String NUM_MASTERS_PROP = "NUM_MASTERS";
- private static final int NUM_TABLET_SERVERS = 3;
- private static final int DEFAULT_NUM_MASTERS = 3;
-
- // Number of masters that will be started for this test if we're starting
- // a cluster.
- private static final int NUM_MASTERS =
- Integer.getInteger(NUM_MASTERS_PROP, DEFAULT_NUM_MASTERS);
-
+ // Default timeout/sleep interval for various client operations, waiting for various jobs/threads
+ // to complete, etc.
protected static final int DEFAULT_SLEEP = 50000;
// Currently not specifying a seed since we want a random behavior when running tests that
@@ -79,28 +72,8 @@ public class BaseKuduTest {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- FakeDNS.getInstance().install();
-
LOG.info("Setting up before class...");
-
- miniCluster = miniClusterBuilder
- .numMasters(NUM_MASTERS)
- .numTservers(NUM_TABLET_SERVERS)
- .defaultTimeoutMs(DEFAULT_SLEEP)
- .build();
-
- masterAddresses = miniCluster.getMasterAddresses();
- masterHostPorts = miniCluster.getMasterHostPorts();
-
- LOG.info("Creating new Kudu client...");
- client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
- .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
- .build();
- syncClient = new KuduClient(client);
- LOG.info("Waiting for tablet servers...");
- if (!miniCluster.waitForTabletServers(NUM_TABLET_SERVERS)) {
- fail("Couldn't get " + NUM_TABLET_SERVERS + " tablet servers running, aborting");
- }
+ doSetup(Integer.getInteger("NUM_MASTERS", 3), 3);
}
@AfterClass
@@ -120,6 +93,39 @@ public class BaseKuduTest {
}
}
+ /**
+ * This method is intended to be called from custom @BeforeClass method to setup Kudu mini cluster
+ * with the specified parameters. The #BaseKuduTest class calls it in its @BeforeClass method
+ * with the default parameters.
+ *
+ * @param numMasters number of masters in the cluster to start
+ * @param numTabletServers number of tablet servers in the cluster to start
+ * @throws Exception if something goes wrong
+ */
+ protected static void doSetup(int numMasters, int numTabletServers)
+ throws Exception {
+ FakeDNS.getInstance().install();
+
+ miniCluster = miniClusterBuilder
+ .numMasters(numMasters)
+ .numTservers(numTabletServers)
+ .defaultTimeoutMs(DEFAULT_SLEEP)
+ .build();
+ masterAddresses = miniCluster.getMasterAddresses();
+ masterHostPorts = miniCluster.getMasterHostPorts();
+
+ LOG.info("Creating new Kudu client...");
+ client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
+ .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
+ .build();
+ syncClient = new KuduClient(client);
+
+ LOG.info("Waiting for tablet servers...");
+ if (!miniCluster.waitForTabletServers(numTabletServers)) {
+ fail(String.format("Couldn't get %d tablet servers running, aborting", numTabletServers));
+ }
+ }
+
protected static KuduTable createTable(String tableName, Schema schema,
CreateTableOptions builder) throws KuduException {
LOG.info("Creating table: {}", tableName);
@@ -246,7 +252,7 @@ public class BaseKuduTest {
}
public static Schema getBasicSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(5);
+ ArrayList<ColumnSchema> columns = new ArrayList<>(5);
columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_i", Type.INT32).build());
@@ -306,14 +312,14 @@ public class BaseKuduTest {
return insert;
}
- static Callback<Object, Object> defaultErrorCB = new Callback<Object, Object>() {
+ static final Callback<Object, Object> defaultErrorCB = new Callback<Object, Object>() {
@Override
public Object call(Object arg) throws Exception {
if (arg == null) return null;
if (arg instanceof Exception) {
LOG.warn("Got exception", (Exception) arg);
} else {
- LOG.warn("Got an error response back " + arg);
+ LOG.warn("Got an error response back {}", arg);
}
return new Exception("Can't recover from error, see previous WARN");
}
@@ -386,8 +392,8 @@ public class BaseKuduTest {
leader = tablet.getLeaderReplica();
if (leader == null) {
- LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept " +
- deadlineTracker.getElapsedMillis() + "ms");
+ LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept {} ms",
+ deadlineTracker.getElapsedMillis());
Thread.sleep(50);
}
}
@@ -424,7 +430,7 @@ public class BaseKuduTest {
.getPort();
}
if (leaderPort == -1) {
- fail("No leader master found after " + DEFAULT_SLEEP + " ms.");
+ fail(String.format("No leader master found after %d ms", DEFAULT_SLEEP));
}
return leaderPort;
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
index 0eaefa5..97d5928 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
@@ -448,8 +448,7 @@ public class ITClient extends BaseKuduTest {
}
}
- class UncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
-
+ private class UncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
@Override
public void uncaughtException(Thread t, Throwable e) {
// Only report an error if we're still running, else we'll spam the log.
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
index f536950..16f9b79 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
@@ -583,8 +583,8 @@ public class MiniKuduCluster implements AutoCloseable {
private int numTservers = 3;
private int defaultTimeoutMs = 50000;
private boolean enableKerberos = false;
- private List<String> extraTserverFlags = new ArrayList<>();
- private List<String> extraMasterFlags = new ArrayList<>();
+ private final List<String> extraTserverFlags = new ArrayList<>();
+ private final List<String> extraMasterFlags = new ArrayList<>();
public MiniKuduClusterBuilder numMasters(int numMasters) {
this.numMasters = numMasters;
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
index 685a01e..8153328 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
@@ -31,8 +31,6 @@ import com.google.protobuf.ByteString;
import com.stumbleupon.async.Deferred;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Common;
@@ -42,8 +40,6 @@ import org.apache.kudu.consensus.Metadata;
import org.apache.kudu.master.Master;
public class TestAsyncKuduClient extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(TestAsyncKuduClient.class);
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
BaseKuduTest.setUpBeforeClass();
@@ -237,7 +233,7 @@ public class TestAsyncKuduClient extends BaseKuduTest {
*/
@Test(timeout = 100000)
public void testCreateTableOutOfOrderPrimaryKeys() throws Exception {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(6);
+ ArrayList<ColumnSchema> columns = new ArrayList<>(6);
columns.add(new ColumnSchema.ColumnSchemaBuilder("key_1", Type.INT8).key(true).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("key_2", Type.INT16).key(true).build());
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
index d84f972..41c7c27 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
@@ -565,12 +565,11 @@ public class TestAsyncKuduSession extends BaseKuduTest {
PartialRow upperBound = schema.newPartialRow();
upperBound.addInt(schema.getColumnByIndex(0).getName(), exclusiveEnd);
- AsyncKuduScanner scanner = client.newScannerBuilder(table)
+ return client.newScannerBuilder(table)
.lowerBound(lowerBound)
.exclusiveUpperBound(upperBound)
.setProjectedColumnNames(columnNames)
.build();
- return scanner;
}
private TabletServerErrorPB makeTabletServerError() {
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index bd455fa..f158c64 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -773,9 +773,7 @@ public class TestKuduClient extends BaseKuduTest {
*/
@Test(timeout = 100000)
public void testScanNonCoveredTable() throws Exception {
-
- Schema schema = basicSchema;
- syncClient.createTable(tableName, schema, getBasicTableOptionsWithNonCoveredRange());
+ syncClient.createTable(tableName, basicSchema, getBasicTableOptionsWithNonCoveredRange());
KuduSession session = syncClient.newSession();
session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
index 273d818..122ed21 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
@@ -32,7 +32,7 @@ import org.junit.rules.TestName;
public class TestKuduSession extends BaseKuduTest {
@Rule
- public TestName name = new TestName();
+ public final TestName name = new TestName();
private KuduTable table;
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
index 39f554c..ac3e52f 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestNegotiator.java
@@ -35,7 +35,6 @@ import javax.net.ssl.SSLEngineResult.HandshakeStatus;
import javax.net.ssl.SSLException;
import javax.security.auth.Subject;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
index 38db309..17706f7 100644
--- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
@@ -30,7 +30,6 @@ import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
index 99aa6d7..937415b 100644
--- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
+++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
@@ -27,7 +27,6 @@ import java.util.HashMap;
import java.util.List;
import com.google.common.base.Charsets;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import org.apache.flume.Channel;
import org.apache.flume.Context;
http://git-wip-us.apache.org/repos/asf/kudu/blob/7e745270/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
index 62e8f58..6037a58 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.List;
import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;