You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2021/03/17 21:58:13 UTC
[GitHub] [kafka] jsancio commented on a change in pull request #10323: KAFKA-12459; Use property testing library for raft event simulation tests
jsancio commented on a change in pull request #10323:
URL: https://github.com/apache/kafka/pull/10323#discussion_r596410292
##########
File path: raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java
##########
@@ -70,335 +108,200 @@
private static final int FETCH_MAX_WAIT_MS = 100;
private static final int LINGER_MS = 0;
- @Test
- public void testInitialLeaderElectionQuorumSizeOne() {
- testInitialLeaderElection(new QuorumConfig(1));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeTwo() {
- testInitialLeaderElection(new QuorumConfig(2));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeThree() {
- testInitialLeaderElection(new QuorumConfig(3));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeFour() {
- testInitialLeaderElection(new QuorumConfig(4));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeFive() {
- testInitialLeaderElection(new QuorumConfig(5));
- }
-
- private void testInitialLeaderElection(QuorumConfig config) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 1);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(10));
- }
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeThree() {
- testElectionAfterLeaderFailure(new QuorumConfig(3, 0));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeThreeAndTwoObservers() {
- testElectionAfterLeaderFailure(new QuorumConfig(3, 1));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFour() {
- testElectionAfterLeaderFailure(new QuorumConfig(4, 0));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFourAndTwoObservers() {
- testElectionAfterLeaderFailure(new QuorumConfig(4, 2));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFive() {
- testElectionAfterLeaderFailure(new QuorumConfig(5, 0));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFiveAndThreeObservers() {
- testElectionAfterLeaderFailure(new QuorumConfig(5, 3));
- }
-
- private void testElectionAfterLeaderFailure(QuorumConfig config) {
- checkElectionAfterLeaderShutdown(config, false);
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeThree() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(3, 0));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeThreeAndTwoObservers() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(3, 2));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFour() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(4, 0));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFourAndTwoObservers() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(4, 2));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFive() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(5, 0));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFiveAndThreeObservers() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(5, 3));
- }
-
- private void checkElectionAfterLeaderGracefulShutdown(QuorumConfig config) {
- checkElectionAfterLeaderShutdown(config, true);
- }
-
- private void checkElectionAfterLeaderShutdown(QuorumConfig config, boolean isGracefulShutdown) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- // Seed the cluster with some data
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 1);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.anyReachedHighWatermark(10));
-
- // Shutdown the leader and write some more data. We can verify the new leader has been elected
- // by verifying that the high watermark can still advance.
- int leaderId = cluster.latestLeader().getAsInt();
- if (isGracefulShutdown) {
- cluster.shutdown(leaderId);
- } else {
- cluster.kill(leaderId);
- }
-
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(20));
- long highWatermark = cluster.maxHighWatermarkReached();
-
- // Restart the node and verify it catches up
- cluster.start(leaderId);
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(highWatermark + 10));
- }
- }
-
- @Test
- public void testRecoveryAfterAllNodesFailQuorumSizeThree() {
- checkRecoveryAfterAllNodesFail(new QuorumConfig(3));
- }
-
- @Test
- public void testRecoveryAfterAllNodesFailQuorumSizeFour() {
- checkRecoveryAfterAllNodesFail(new QuorumConfig(4));
- }
-
- @Test
- public void testRecoveryAfterAllNodesFailQuorumSizeFive() {
- checkRecoveryAfterAllNodesFail(new QuorumConfig(5));
- }
-
- private void checkRecoveryAfterAllNodesFail(QuorumConfig config) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- // Seed the cluster with some data
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 1);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.anyReachedHighWatermark(10));
- long highWatermark = cluster.maxHighWatermarkReached();
-
- // We kill all of the nodes. Then we bring back a majority and verify that
- // they are able to elect a leader and continue making progress
-
- cluster.killAll();
-
- Iterator<Integer> nodeIdsIterator = cluster.nodes().iterator();
- for (int i = 0; i < cluster.majoritySize(); i++) {
- Integer nodeId = nodeIdsIterator.next();
- cluster.start(nodeId);
- }
-
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(highWatermark + 10));
- }
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeThree() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(3));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeThreeAndTwoObservers() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(3, 2));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFour() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(4));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFourAndTwoObservers() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(4, 2));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFive() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(5));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFiveAndThreeObservers() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(5, 3));
- }
-
- private void checkElectionAfterLeaderNetworkPartition(QuorumConfig config) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- // Seed the cluster with some data
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 2);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.anyReachedHighWatermark(10));
-
- // The leader gets partitioned off. We can verify the new leader has been elected
- // by writing some data and ensuring that it gets replicated
- int leaderId = cluster.latestLeader().getAsInt();
- router.filter(leaderId, new DropAllTraffic());
-
- Set<Integer> nonPartitionedNodes = new HashSet<>(cluster.nodes());
- nonPartitionedNodes.remove(leaderId);
-
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(20, nonPartitionedNodes));
- }
+ @Property(tries = 100)
+ void canElectInitialLeader(
+ @ForAll Random random,
Review comment:
I assume that `@ForAll` will cause the library to run this test multiple times with different random seeds. If so is there a way to tell the library to only run it once for some random seed?
##########
File path: raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java
##########
@@ -70,335 +108,200 @@
private static final int FETCH_MAX_WAIT_MS = 100;
private static final int LINGER_MS = 0;
- @Test
- public void testInitialLeaderElectionQuorumSizeOne() {
- testInitialLeaderElection(new QuorumConfig(1));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeTwo() {
- testInitialLeaderElection(new QuorumConfig(2));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeThree() {
- testInitialLeaderElection(new QuorumConfig(3));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeFour() {
- testInitialLeaderElection(new QuorumConfig(4));
- }
-
- @Test
- public void testInitialLeaderElectionQuorumSizeFive() {
- testInitialLeaderElection(new QuorumConfig(5));
- }
-
- private void testInitialLeaderElection(QuorumConfig config) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 1);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(10));
- }
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeThree() {
- testElectionAfterLeaderFailure(new QuorumConfig(3, 0));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeThreeAndTwoObservers() {
- testElectionAfterLeaderFailure(new QuorumConfig(3, 1));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFour() {
- testElectionAfterLeaderFailure(new QuorumConfig(4, 0));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFourAndTwoObservers() {
- testElectionAfterLeaderFailure(new QuorumConfig(4, 2));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFive() {
- testElectionAfterLeaderFailure(new QuorumConfig(5, 0));
- }
-
- @Test
- public void testElectionAfterLeaderFailureQuorumSizeFiveAndThreeObservers() {
- testElectionAfterLeaderFailure(new QuorumConfig(5, 3));
- }
-
- private void testElectionAfterLeaderFailure(QuorumConfig config) {
- checkElectionAfterLeaderShutdown(config, false);
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeThree() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(3, 0));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeThreeAndTwoObservers() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(3, 2));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFour() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(4, 0));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFourAndTwoObservers() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(4, 2));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFive() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(5, 0));
- }
-
- @Test
- public void testElectionAfterLeaderGracefulShutdownQuorumSizeFiveAndThreeObservers() {
- checkElectionAfterLeaderGracefulShutdown(new QuorumConfig(5, 3));
- }
-
- private void checkElectionAfterLeaderGracefulShutdown(QuorumConfig config) {
- checkElectionAfterLeaderShutdown(config, true);
- }
-
- private void checkElectionAfterLeaderShutdown(QuorumConfig config, boolean isGracefulShutdown) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- // Seed the cluster with some data
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 1);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.anyReachedHighWatermark(10));
-
- // Shutdown the leader and write some more data. We can verify the new leader has been elected
- // by verifying that the high watermark can still advance.
- int leaderId = cluster.latestLeader().getAsInt();
- if (isGracefulShutdown) {
- cluster.shutdown(leaderId);
- } else {
- cluster.kill(leaderId);
- }
-
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(20));
- long highWatermark = cluster.maxHighWatermarkReached();
-
- // Restart the node and verify it catches up
- cluster.start(leaderId);
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(highWatermark + 10));
- }
- }
-
- @Test
- public void testRecoveryAfterAllNodesFailQuorumSizeThree() {
- checkRecoveryAfterAllNodesFail(new QuorumConfig(3));
- }
-
- @Test
- public void testRecoveryAfterAllNodesFailQuorumSizeFour() {
- checkRecoveryAfterAllNodesFail(new QuorumConfig(4));
- }
-
- @Test
- public void testRecoveryAfterAllNodesFailQuorumSizeFive() {
- checkRecoveryAfterAllNodesFail(new QuorumConfig(5));
- }
-
- private void checkRecoveryAfterAllNodesFail(QuorumConfig config) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- // Seed the cluster with some data
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 1);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.anyReachedHighWatermark(10));
- long highWatermark = cluster.maxHighWatermarkReached();
-
- // We kill all of the nodes. Then we bring back a majority and verify that
- // they are able to elect a leader and continue making progress
-
- cluster.killAll();
-
- Iterator<Integer> nodeIdsIterator = cluster.nodes().iterator();
- for (int i = 0; i < cluster.majoritySize(); i++) {
- Integer nodeId = nodeIdsIterator.next();
- cluster.start(nodeId);
- }
-
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(highWatermark + 10));
- }
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeThree() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(3));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeThreeAndTwoObservers() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(3, 2));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFour() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(4));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFourAndTwoObservers() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(4, 2));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFive() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(5));
- }
-
- @Test
- public void testElectionAfterLeaderNetworkPartitionQuorumSizeFiveAndThreeObservers() {
- checkElectionAfterLeaderNetworkPartition(new QuorumConfig(5, 3));
- }
-
- private void checkElectionAfterLeaderNetworkPartition(QuorumConfig config) {
- for (int seed = 0; seed < 100; seed++) {
- Cluster cluster = new Cluster(config, seed);
- MessageRouter router = new MessageRouter(cluster);
- EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
-
- // Seed the cluster with some data
- cluster.startAll();
- schedulePolling(scheduler, cluster, 3, 5);
- scheduler.schedule(router::deliverAll, 0, 2, 2);
- scheduler.schedule(new SequentialAppendAction(cluster), 0, 2, 3);
- scheduler.runUntil(cluster::hasConsistentLeader);
- scheduler.runUntil(() -> cluster.anyReachedHighWatermark(10));
-
- // The leader gets partitioned off. We can verify the new leader has been elected
- // by writing some data and ensuring that it gets replicated
- int leaderId = cluster.latestLeader().getAsInt();
- router.filter(leaderId, new DropAllTraffic());
-
- Set<Integer> nonPartitionedNodes = new HashSet<>(cluster.nodes());
- nonPartitionedNodes.remove(leaderId);
-
- scheduler.runUntil(() -> cluster.allReachedHighWatermark(20, nonPartitionedNodes));
- }
+ @Property(tries = 100)
Review comment:
Hmm. These simulation tests are already the slowest suite in `raft:test`. I `git fetch` the PR and it looks like they still run when calling `./gradlew raft:test`. Is there away to set this to a smaller number (2) as the default and override it if the user wants to run them with 100 tries?
Btw, `5` (number of possible voters) * `6` (number of possible observers) is `30` which is much smaller than `100`. I assume that the library will run the same configuration with multiple seeds.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org