You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by ni...@apache.org on 2022/07/21 09:35:35 UTC
[bookkeeper] branch master updated: Issue 3206: Flaky-test: BookieZKExpireTest.testBookieServerZKExpireBehaviour (#3415)
This is an automated email from the ASF dual-hosted git repository.
nicoloboschi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 0b910db875 Issue 3206: Flaky-test: BookieZKExpireTest.testBookieServerZKExpireBehaviour (#3415)
0b910db875 is described below
commit 0b910db875e6c5f821e69c41330aa3e2686ae224
Author: Andrey Yegorov <86...@users.noreply.github.com>
AuthorDate: Thu Jul 21 02:35:30 2022 -0700
Issue 3206: Flaky-test: BookieZKExpireTest.testBookieServerZKExpireBehaviour (#3415)
* Issue 3206: Flaky-test: BookieZKExpireTest.testBookieServerZKExpireBehaviour
* clear property at the end of the test
---
.../bookkeeper/bookie/BookieStateManager.java | 10 ++-
.../apache/bookkeeper/test/BookieZKExpireTest.java | 95 ++++++++++++++++++++--
2 files changed, 94 insertions(+), 11 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
index 3d4c0f30a0..a6f76a604f 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
@@ -118,10 +118,11 @@ public class BookieStateManager implements StateManager {
this.rm = rm;
if (this.rm != null) {
rm.addRegistrationListener(() -> {
- forceToUnregistered();
- // schedule a re-register operation
- registerBookie(false);
- });
+ log.info("Trying to re-register the bookie");
+ forceToUnregistered();
+ // schedule a re-register operation
+ registerBookie(false);
+ });
}
this.statusDirs = statusDirs;
@@ -228,6 +229,7 @@ public class BookieStateManager implements StateManager {
@Override
public Void call() throws IOException {
try {
+ log.info("Re-registering the bookie");
doRegisterBookie();
} catch (IOException ioe) {
if (throwException) {
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
index 006c485eee..fae912d54a 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
@@ -21,6 +21,7 @@
package org.apache.bookkeeper.test;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -30,6 +31,7 @@ import io.netty.buffer.UnpooledByteBufAllocator;
import java.io.File;
import java.util.HashSet;
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.bookie.MockUncleanShutdownDetection;
import org.apache.bookkeeper.bookie.TestBookieImpl;
import org.apache.bookkeeper.conf.ServerConfiguration;
@@ -41,18 +43,93 @@ import org.junit.Test;
/**
* Test bookie expiration.
*/
+@Slf4j
public class BookieZKExpireTest extends BookKeeperClusterTestCase {
public BookieZKExpireTest() {
super(0);
- // 6000 is minimum due to default tick time
- baseConf.setZkTimeout(6000);
- baseClientConf.setZkTimeout(6000);
}
+ /*
+ Should recover from request timeout.
+ */
+ @Test
@SuppressWarnings("deprecation")
+ public void testBookieServerZKRequestTimeoutBehaviour() throws Exception {
+ // 6000 is minimum due to default tick time
+ System.setProperty("zookeeper.request.timeout", "6000");
+ baseConf.setZkTimeout(24000);
+ baseClientConf.setZkTimeout(24000);
+ BookieServer server = null;
+ try {
+ File f = tmpDirs.createNew("bookieserver", "test");
+
+ HashSet<Thread> threadset = new HashSet<Thread>();
+ int threadCount = Thread.activeCount();
+ Thread[] threads = new Thread[threadCount * 2];
+ threadCount = Thread.enumerate(threads);
+ for (int i = 0; i < threadCount; i++) {
+ if (threads[i].getName().indexOf("SendThread") != -1) {
+ threadset.add(threads[i]);
+ }
+ }
+
+ ServerConfiguration conf = newServerConfiguration(PortManager.nextFreePort(), f, new File[] { f });
+ server = new BookieServer(
+ conf, new TestBookieImpl(conf),
+ NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
+ new MockUncleanShutdownDetection());
+ server.start();
+
+ int secondsToWait = 5;
+ while (!server.isRunning()) {
+ Thread.sleep(1000);
+ if (secondsToWait-- <= 0) {
+ fail("Bookie never started");
+ }
+ }
+ Thread sendthread = null;
+ threadCount = Thread.activeCount();
+ threads = new Thread[threadCount * 2];
+ threadCount = Thread.enumerate(threads);
+ for (int i = 0; i < threadCount; i++) {
+ if (threads[i].getName().indexOf("SendThread") != -1
+ && !threadset.contains(threads[i])) {
+ sendthread = threads[i];
+ break;
+ }
+ }
+ assertNotNull("Send thread not found", sendthread);
+
+ log.info("Suspending threads");
+ sendthread.suspend();
+ Thread.sleep(12000);
+ log.info("Resuming threads");
+ sendthread.resume();
+
+ // allow watcher thread to run
+ Thread.sleep(3000);
+ assertTrue("Bookie should not shutdown on zk timeout", server.isBookieRunning());
+ assertTrue("Bookie Server should not shutdown on zk timeout", server.isRunning());
+ } finally {
+ System.clearProperty("zookeeper.request.timeout");
+ server.shutdown();
+ }
+ }
+
+ /*
+ Bookie cannot recover from ZK Client's SessionExpired error.
+ In this case the ZK client must be recreated, reconnect does not work.
+ Attempt to reconnect by BookieStateManager's RegistrationManager listener
+ will fail (even if retry it many times).
+ */
@Test
- public void testBookieServerZKExpireBehaviour() throws Exception {
+ @SuppressWarnings("deprecation")
+ public void testBookieServerZKSessionExpireBehaviour() throws Exception {
+ // 6000 is minimum due to default tick time
+ System.setProperty("zookeeper.request.timeout", "0");
+ baseConf.setZkTimeout(6000);
+ baseClientConf.setZkTimeout(6000);
BookieServer server = null;
try {
File f = tmpDirs.createNew("bookieserver", "test");
@@ -94,16 +171,20 @@ public class BookieZKExpireTest extends BookKeeperClusterTestCase {
}
assertNotNull("Send thread not found", sendthread);
+ log.info("Suspending threads");
sendthread.suspend();
- Thread.sleep(2 * conf.getZkTimeout());
+ Thread.sleep(2L * conf.getZkTimeout());
+ log.info("Resuming threads");
sendthread.resume();
// allow watcher thread to run
Thread.sleep(3000);
- assertTrue("Bookie should not shutdown on losing zk session", server.isBookieRunning());
- assertTrue("Bookie Server should not shutdown on losing zk session", server.isRunning());
+ assertFalse("Bookie should shutdown on losing zk session", server.isBookieRunning());
+ assertFalse("Bookie Server should shutdown on losing zk session", server.isRunning());
} finally {
+ System.clearProperty("zookeeper.request.timeout");
server.shutdown();
}
}
+
}