You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by lh...@apache.org on 2025/11/28 07:36:27 UTC
(bookkeeper) branch branch-4.16 updated (35b8f137b4 -> dd0ed9d65b)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a change to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from 35b8f137b4 Log all the error in the GarbageCollectorThread (#4649)
new 2b2661de68 [fix]Wrong error code(-107) of opening a deleted ledger (#4657)
new c398379658 [fix]BK stays at read_only state even if the disk is empty (#4640)
new dd0ed9d65b Bring back the old public method name (#4682)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/bookkeeper/bookie/BookieException.java | 12 ++
.../bookkeeper/bookie/BookieStateManager.java | 29 ++--
.../org/apache/bookkeeper/bookie/BookieStatus.java | 19 ++-
.../bookkeeper/bookie/HandleFactoryImpl.java | 2 +-
.../bookkeeper/bookie/LedgerDirsMonitor.java | 17 ++-
.../org/apache/bookkeeper/bookie/StateManager.java | 12 +-
.../bookkeeper/proto/PerChannelBookieClient.java | 5 +
.../bookkeeper/proto/ReadEntryProcessor.java | 2 +-
.../bookkeeper/proto/ReadEntryProcessorV3.java | 5 +-
.../bookkeeper/proto/WriteEntryProcessor.java | 4 +-
.../bookkeeper/proto/WriteEntryProcessorV3.java | 4 +-
.../bookkeeper/proto/WriteLacProcessorV3.java | 4 +
.../http/service/BookieStateReadOnlyService.java | 4 +-
.../bookie/BookieInitializationTest.java | 153 +++++++++++++++++++--
.../apache/bookkeeper/bookie/LedgerCacheTest.java | 2 +-
.../org/apache/bookkeeper/client/TestFencing.java | 113 +++++++++++++++
.../replication/AuditorLedgerCheckerTest.java | 4 +-
17 files changed, 353 insertions(+), 38 deletions(-)
(bookkeeper) 03/03: Bring back the old public method name (#4682)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit dd0ed9d65bdb88b39bcee5030e7fa024c939f48a
Author: Yong Zhang <zh...@gmail.com>
AuthorDate: Mon Nov 24 11:32:24 2025 +0800
Bring back the old public method name (#4682)
(cherry picked from commit 1a509e4f736a75bc4570748324afba94ba51e471)
---
.../java/org/apache/bookkeeper/bookie/BookieStateManager.java | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
index c50d51a96a..070897b947 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
@@ -297,6 +297,10 @@ public class BookieStateManager implements StateManager {
}
}
+ public void doTransitionToWritableMode() {
+ doTransitionToWritableMode(false);
+ }
+
@VisibleForTesting
public void doTransitionToWritableMode(boolean isManuallyModify) {
if (shuttingdown || forceReadOnly.get()) {
@@ -338,6 +342,10 @@ public class BookieStateManager implements StateManager {
}
}
+ public void doTransitionToReadOnlyMode() {
+ doTransitionToReadOnlyMode(false);
+ }
+
@VisibleForTesting
public void doTransitionToReadOnlyMode(boolean isManuallyModify) {
if (shuttingdown) {
(bookkeeper) 02/03: [fix]BK stays at read_only state even if the disk is empty (#4640)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit c39837965872572d45293578decc9081fafc04bb
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Wed Sep 17 12:29:19 2025 +0800
[fix]BK stays at read_only state even if the disk is empty (#4640)
* [fix]BK stay at read_only state even if the disk is empty
* correct the javadoc
* improve test
(cherry picked from commit bde9ff2503ae71c4ef03b212577a1a8be673123a)
---
.../bookkeeper/bookie/BookieStateManager.java | 21 +--
.../org/apache/bookkeeper/bookie/BookieStatus.java | 19 ++-
.../bookkeeper/bookie/LedgerDirsMonitor.java | 17 ++-
.../org/apache/bookkeeper/bookie/StateManager.java | 12 +-
.../http/service/BookieStateReadOnlyService.java | 4 +-
.../bookie/BookieInitializationTest.java | 153 +++++++++++++++++++--
.../apache/bookkeeper/bookie/LedgerCacheTest.java | 2 +-
.../replication/AuditorLedgerCheckerTest.java | 4 +-
8 files changed, 202 insertions(+), 30 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
index 620a1c2aaa..c50d51a96a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
@@ -165,7 +165,7 @@ public class BookieStateManager implements StateManager {
@Override
public void initState(){
if (forceReadOnly.get()) {
- this.bookieStatus.setToReadOnlyMode();
+ this.bookieStatus.setToReadOnlyMode(false);
} else if (conf.isPersistBookieStatusEnabled()) {
this.bookieStatus.readFromDirectories(statusDirs);
}
@@ -256,22 +256,22 @@ public class BookieStateManager implements StateManager {
}
@Override
- public Future<Void> transitionToWritableMode() {
+ public Future<Void> transitionToWritableMode(boolean isManuallyModify) {
return stateService.submit(new Callable<Void>() {
@Override
public Void call() throws Exception{
- doTransitionToWritableMode();
+ doTransitionToWritableMode(isManuallyModify);
return null;
}
});
}
@Override
- public Future<Void> transitionToReadOnlyMode() {
+ public Future<Void> transitionToReadOnlyMode(boolean isManuallyModify) {
return stateService.submit(new Callable<Void>() {
@Override
public Void call() throws Exception{
- doTransitionToReadOnlyMode();
+ doTransitionToReadOnlyMode(isManuallyModify);
return null;
}
});
@@ -298,10 +298,15 @@ public class BookieStateManager implements StateManager {
}
@VisibleForTesting
- public void doTransitionToWritableMode() {
+ public void doTransitionToWritableMode(boolean isManuallyModify) {
if (shuttingdown || forceReadOnly.get()) {
return;
}
+ if (!isManuallyModify && bookieStatus.isInReadOnlyMode() && bookieStatus.isManuallyModifiedToReadOnly()) {
+ LOG.info("Skip to transition Bookie to Writable mode automatically because it is manually set to read-only"
+ + " mode, which can only be changed manually.");
+ return;
+ }
if (!bookieStatus.setToWritableMode()) {
// do nothing if already in writable mode
@@ -334,11 +339,11 @@ public class BookieStateManager implements StateManager {
}
@VisibleForTesting
- public void doTransitionToReadOnlyMode() {
+ public void doTransitionToReadOnlyMode(boolean isManuallyModify) {
if (shuttingdown) {
return;
}
- if (!bookieStatus.setToReadOnlyMode()) {
+ if (!bookieStatus.setToReadOnlyMode(isManuallyModify)) {
return;
}
if (!conf.isReadOnlyModeEnabled()) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java
index 7b2d0aad99..ccbba36b53 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java
@@ -31,6 +31,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.List;
+import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +54,8 @@ public class BookieStatus {
private int layoutVersion;
private long lastUpdateTime;
private volatile BookieMode bookieMode;
+ @Getter
+ private boolean isManuallyModifiedToReadOnly;
BookieStatus() {
this.bookieMode = BookieMode.READ_WRITE;
@@ -72,6 +75,7 @@ public class BookieStatus {
if (!bookieMode.equals(BookieMode.READ_WRITE)) {
bookieMode = BookieMode.READ_WRITE;
this.lastUpdateTime = System.currentTimeMillis();
+ this.isManuallyModifiedToReadOnly = false;
return true;
}
return false;
@@ -81,10 +85,11 @@ public class BookieStatus {
return bookieMode.equals(BookieMode.READ_ONLY);
}
- synchronized boolean setToReadOnlyMode() {
+ synchronized boolean setToReadOnlyMode(boolean isManuallyModify) {
if (!bookieMode.equals(BookieMode.READ_ONLY)) {
bookieMode = BookieMode.READ_ONLY;
this.lastUpdateTime = System.currentTimeMillis();
+ this.isManuallyModifiedToReadOnly = isManuallyModify;
return true;
}
return false;
@@ -147,6 +152,7 @@ public class BookieStatus {
this.lastUpdateTime = status.lastUpdateTime;
this.layoutVersion = status.layoutVersion;
this.bookieMode = status.bookieMode;
+ this.isManuallyModifiedToReadOnly = status.isManuallyModifiedToReadOnly;
success = true;
}
}
@@ -216,6 +222,15 @@ public class BookieStatus {
if (status.layoutVersion == 1 && parts.length == 3) {
status.bookieMode = BookieMode.valueOf(parts[1]);
status.lastUpdateTime = Long.parseLong(parts[2].trim());
+ status.isManuallyModifiedToReadOnly = true;
+ return status;
+ }
+ // Since we should guarantee the compatibility of downgrade. We do not change the layoutVersion, otherwise,
+ // the string can not be parsed by the lower version.
+ if (status.layoutVersion == 1 && parts.length == 4) {
+ status.bookieMode = BookieMode.valueOf(parts[1]);
+ status.lastUpdateTime = Long.parseLong(parts[2].trim());
+ status.isManuallyModifiedToReadOnly = Boolean.parseBoolean(parts[3].trim());
return status;
}
}
@@ -231,6 +246,8 @@ public class BookieStatus {
builder.append(getBookieMode());
builder.append(",");
builder.append(System.currentTimeMillis());
+ builder.append(",");
+ builder.append(isManuallyModifiedToReadOnly);
builder.append("\n");
return builder.toString();
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
index 7ad8ba1e0c..73e2ba1871 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
@@ -21,6 +21,7 @@
package org.apache.bookkeeper.bookie;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.File;
import java.io.IOException;
@@ -54,6 +55,7 @@ class LedgerDirsMonitor {
private final long minUsableSizeForHighPriorityWrites;
private ScheduledExecutorService executor;
private ScheduledFuture<?> checkTask;
+ private boolean isFirstLoopOfCheckTask = true;
public LedgerDirsMonitor(final ServerConfiguration conf,
final DiskChecker diskChecker,
@@ -67,6 +69,10 @@ class LedgerDirsMonitor {
}
private void check(final LedgerDirsManager ldm) {
+ final boolean isFirstLoopOfCheckTaskLocalValue = this.isFirstLoopOfCheckTask;
+ if (isFirstLoopOfCheckTaskLocalValue) {
+ this.isFirstLoopOfCheckTask = false;
+ }
final ConcurrentMap<File, Float> diskUsages = ldm.getDiskUsages();
boolean someDiskFulled = false;
boolean highPriorityWritesAllowed = true;
@@ -175,6 +181,14 @@ class LedgerDirsMonitor {
}
}
+ if (isFirstLoopOfCheckTaskLocalValue && ldm.getFullFilledLedgerDirs().isEmpty()) {
+ // notify no disk full.
+ for (LedgerDirsListener listener : ldm.getListeners()) {
+ listener.allDisksWritable();
+ }
+ return;
+ }
+
if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
if (someDiskFulled && !ldm.getFullFilledLedgerDirs().isEmpty()) {
// notify any disk full.
@@ -192,7 +206,8 @@ class LedgerDirsMonitor {
}
}
- private void check() {
+ @VisibleForTesting
+ void check() {
dirsManagers.forEach(this::check);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java
index 7ed3f0b657..9ef1f384e7 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java
@@ -100,12 +100,20 @@ public interface StateManager extends AutoCloseable {
/**
* Change the state of bookie to Writable mode.
*/
- Future<Void> transitionToWritableMode();
+ Future<Void> transitionToWritableMode(boolean isManuallyModify);
+
+ default Future<Void> transitionToWritableMode() {
+ return transitionToWritableMode(false);
+ }
/**
* Change the state of bookie to ReadOnly mode.
*/
- Future<Void> transitionToReadOnlyMode();
+ Future<Void> transitionToReadOnlyMode(boolean isManuallyModify);
+
+ default Future<Void> transitionToReadOnlyMode() {
+ return transitionToReadOnlyMode(false);
+ }
/**
* ShutdownHandler used to shutdown bookie.
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java
index d32074e2bb..73f57c1959 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java
@@ -58,14 +58,14 @@ public class BookieStateReadOnlyService implements HttpEndpointService {
response.setBody("Bookie is in forceReadOnly mode, cannot transit to writable mode");
return response;
}
- stateManager.transitionToWritableMode().get();
+ stateManager.transitionToWritableMode(true).get();
} else if (!stateManager.isReadOnly() && inState.isReadOnly()) {
if (!stateManager.isReadOnlyModeEnabled()) {
response.setCode(HttpServer.StatusCode.BAD_REQUEST);
response.setBody("Bookie is disabled ReadOnly mode, cannot transit to readOnly mode");
return response;
}
- stateManager.transitionToReadOnlyMode().get();
+ stateManager.transitionToReadOnlyMode(true).get();
}
} else if (!HttpServer.Method.GET.equals(request.getMethod())) {
response.setCode(HttpServer.StatusCode.NOT_FOUND);
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
index 5188eb816b..bd0a12ca6c 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
@@ -77,6 +77,7 @@ import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.common.component.ComponentStarter;
import org.apache.bookkeeper.common.component.Lifecycle;
import org.apache.bookkeeper.common.component.LifecycleComponent;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
@@ -84,7 +85,10 @@ import org.apache.bookkeeper.discover.BookieServiceInfo;
import org.apache.bookkeeper.discover.BookieServiceInfo.Endpoint;
import org.apache.bookkeeper.discover.RegistrationManager;
import org.apache.bookkeeper.http.HttpRouter;
+import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.HttpServerLoader;
+import org.apache.bookkeeper.http.service.HttpServiceRequest;
+import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.meta.MetadataBookieDriver;
import org.apache.bookkeeper.meta.exceptions.MetadataException;
import org.apache.bookkeeper.meta.zk.ZKMetadataBookieDriver;
@@ -96,6 +100,7 @@ import org.apache.bookkeeper.replication.AutoRecoveryMain;
import org.apache.bookkeeper.replication.ReplicationStats;
import org.apache.bookkeeper.server.Main;
import org.apache.bookkeeper.server.conf.BookieConfiguration;
+import org.apache.bookkeeper.server.http.service.BookieStateReadOnlyService;
import org.apache.bookkeeper.server.service.AutoRecoveryService;
import org.apache.bookkeeper.server.service.BookieService;
import org.apache.bookkeeper.stats.NullStatsLogger;
@@ -107,8 +112,10 @@ import org.apache.bookkeeper.util.PortManager;
import org.apache.bookkeeper.versioning.Version;
import org.apache.bookkeeper.versioning.Versioned;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
+import org.apache.commons.io.FileUtils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
+import org.awaitility.Awaitility;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
@@ -1445,30 +1452,150 @@ public class BookieInitializationTest extends BookKeeperClusterTestCase {
.setPersistBookieStatusEnabled(true)
.setMetadataServiceUri(metadataServiceUri);
// start a new bookie
- BookieServer bookieServer = new BookieServer(
+ BookieServer bookieServer1 = new BookieServer(
conf, new TestBookieImpl(conf),
NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
new MockUncleanShutdownDetection());
- bookieServer.start();
+ bookieServer1.start();
// transition in to read only and persist the status on disk
- Bookie bookie = (BookieImpl) bookieServer.getBookie();
- assertFalse(bookie.isReadOnly());
- bookie.getStateManager().transitionToReadOnlyMode().get();
- assertTrue(bookie.isReadOnly());
+ Bookie bookie1 = (BookieImpl) bookieServer1.getBookie();
+ assertFalse(bookie1.isReadOnly());
+ bookie1.getStateManager().transitionToReadOnlyMode().get();
+ assertTrue(bookie1.isReadOnly());
// corrupt status file
- List<File> ledgerDirs = ((BookieImpl) bookie).getLedgerDirsManager().getAllLedgerDirs();
+ List<File> ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs();
corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME));
corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME));
// restart the bookie should be in read only mode
- bookieServer.shutdown();
- bookieServer = new BookieServer(
+ bookieServer1.shutdown();
+ BookieServer bookieServer2 = new BookieServer(
conf, new TestBookieImpl(conf),
NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
new MockUncleanShutdownDetection());
- bookieServer.start();
- bookie = bookieServer.getBookie();
- assertTrue(bookie.isReadOnly());
- bookieServer.shutdown();
+ bookieServer2.start();
+ BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie();
+ assertTrue(bookie2.isReadOnly());
+ // After a disk check, the bookie should switch to writable mode because the disk usage is fine.
+ bookie2.dirsMonitor.check();
+ Awaitility.await().untilAsserted(() -> {
+ assertFalse(bookie2.isReadOnly());
+ });
+ bookieServer2.shutdown();
+ }
+
+
+ private void setBookieToReadOnly(Bookie bookie, boolean readOnly) throws Exception {
+ BookieStateReadOnlyService.ReadOnlyState state = new BookieStateReadOnlyService.ReadOnlyState();
+ state.setReadOnly(readOnly);
+ HttpServiceRequest request = new HttpServiceRequest(JsonUtil.toJson(state), HttpServer.Method.PUT, null);
+ BookieStateReadOnlyService service = new BookieStateReadOnlyService(bookie);
+ HttpServiceResponse response1 = service.handle(request);
+ assertEquals(HttpServer.StatusCode.OK.getValue(), response1.getStatusCode());
+ }
+
+ /**
+ * Verify: once the state is manually modified to read-only by Admin API, it should not be changed to writable
+ * by the monitor task.
+ * But it can be changed to read-only by monitor task if it was manually set to writable by Admin API.
+ */
+ @Test(timeout = 1000 * 30)
+ public void testRetrieveBookieStatusAdminModifiedWhenStatusFileIsCorrupted() throws Exception {
+ File[] tmpLedgerDirs = new File[3];
+ String[] filePath = new String[tmpLedgerDirs.length];
+ for (int i = 0; i < tmpLedgerDirs.length; i++) {
+ tmpLedgerDirs[i] = tmpDirs.createNew("bookie", "test" + i);
+ filePath[i] = tmpLedgerDirs[i].getPath();
+ }
+ final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
+ conf.setJournalDirName(filePath[0])
+ .setLedgerDirNames(filePath)
+ .setReadOnlyModeEnabled(true)
+ .setPersistBookieStatusEnabled(true)
+ .setMetadataServiceUri(metadataServiceUri);
+ // start a new bookie
+ BookieServer bookieServer1 = new BookieServer(
+ conf, new TestBookieImpl(conf),
+ NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
+ new MockUncleanShutdownDetection());
+ bookieServer1.start();
+ // transition in to read only and persist the status on disk
+ Bookie bookie1 = (BookieImpl) bookieServer1.getBookie();
+ assertFalse(bookie1.isReadOnly());
+ setBookieToReadOnly(bookie1, true);
+ assertTrue(bookie1.isReadOnly());
+ // corrupt status file
+ List<File> ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs();
+ corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME));
+ corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME));
+ // restart the bookie should be in read only mode
+ bookieServer1.shutdown();
+ BookieServer bookieServer2 = new BookieServer(
+ conf, new TestBookieImpl(conf),
+ NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
+ new MockUncleanShutdownDetection());
+ bookieServer2.start();
+ BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie();
+ assertTrue(bookie2.isReadOnly());
+ // After a disk check, the bookie should not switch to writable mode because the state was set to read-only
+ // by admin api manually.
+ bookie2.dirsMonitor.check();
+ Thread.sleep(2000);
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue(bookie2.isReadOnly());
+ });
+ // We can use Admin Api to change the state back to writable.
+ setBookieToReadOnly(bookie2, false);
+ assertFalse(bookie2.isReadOnly());
+ // The state can be changed to read-only by monitor task if it was manually set to writable by Admin API.
+ bookie2.getStateManager().transitionToReadOnlyMode().get();
+ assertTrue(bookie2.isReadOnly());
+ bookieServer2.shutdown();
+ }
+
+ /**
+ * Verify: the newest version can read the old version payload of persisted bookie status.
+ * old payload: "1,READ_ONLY,1752809349613"
+ * new payload: "1,READ_ONLY,1752809349613,false"
+ */
+ @Test(timeout = 30000)
+ public void testPersistBookieStatusCompatibility() throws Exception {
+ File[] tmpLedgerDirs = new File[1];
+ String[] filePath = new String[1];
+ tmpLedgerDirs[0] = tmpDirs.createNew("bookie", "test");
+ filePath[0] = tmpLedgerDirs[0].getPath();
+ final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
+ conf.setJournalDirName(filePath[0])
+ .setLedgerDirNames(filePath)
+ .setReadOnlyModeEnabled(true)
+ .setPersistBookieStatusEnabled(true)
+ .setMetadataServiceUri(metadataServiceUri);
+ // start a new bookie
+ BookieServer bookieServer1 = new BookieServer(
+ conf, new TestBookieImpl(conf),
+ NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
+ new MockUncleanShutdownDetection());
+ bookieServer1.start();
+ Bookie bookie1 = (BookieImpl) bookieServer1.getBookie();
+ // restart the bookie should be in read only mode
+ bookieServer1.shutdown();
+ // Rewrite file to the old version payload.
+ List<File> ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs();
+ FileUtils.writeStringToFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME),
+ "1,READ_ONLY,1752809349613");
+ BookieServer bookieServer2 = new BookieServer(
+ conf, new TestBookieImpl(conf),
+ NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT,
+ new MockUncleanShutdownDetection());
+ bookieServer2.start();
+ BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie();
+ assertTrue(bookie2.isReadOnly());
+ // After a disk check, the bookie should not switch to writable mode because the state was set to read-only
+ // by admin api manually(the default value of previous version is true).
+ bookie2.dirsMonitor.check();
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue(bookie2.isReadOnly());
+ });
+ bookieServer2.shutdown();
}
/**
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
index 4e8f06fe7f..9e2090d797 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
@@ -711,7 +711,7 @@ public class LedgerCacheTest {
LOG.info("Started flushing mem table.");
memTable.flush(FlushTestSortedLedgerStorage.this);
} catch (IOException e) {
- getStateManager().doTransitionToReadOnlyMode();
+ getStateManager().doTransitionToReadOnlyMode(false);
LOG.error("Exception thrown while flushing skip list cache.", e);
}
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
index 2e3e09012f..bbeb7e758f 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
@@ -326,7 +326,7 @@ public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase {
BookieServer bk = serverByIndex(bkIndex);
bookieConf.setReadOnlyModeEnabled(true);
- ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode();
+ ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false);
bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex)))
.get(30, TimeUnit.SECONDS);
@@ -361,7 +361,7 @@ public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase {
BookieServer bk = serverByIndex(bkIndex);
bookieConf.setReadOnlyModeEnabled(true);
- ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode();
+ ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false);
bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex)))
.get(30, TimeUnit.SECONDS);
(bookkeeper) 01/03: [fix]Wrong error code(-107) of opening a deleted ledger (#4657)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 2b2661de68ad0d9b02fd964c1520def07d044911
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Tue Sep 2 12:26:50 2025 +0800
[fix]Wrong error code(-107) of opening a deleted ledger (#4657)
(cherry picked from commit 5870922754b9585f7d3592dcd0714a4da65d9d80)
---
.../apache/bookkeeper/bookie/BookieException.java | 12 +++
.../bookkeeper/bookie/HandleFactoryImpl.java | 2 +-
.../bookkeeper/proto/PerChannelBookieClient.java | 5 +
.../bookkeeper/proto/ReadEntryProcessor.java | 2 +-
.../bookkeeper/proto/ReadEntryProcessorV3.java | 5 +-
.../bookkeeper/proto/WriteEntryProcessor.java | 4 +-
.../bookkeeper/proto/WriteEntryProcessorV3.java | 4 +-
.../bookkeeper/proto/WriteLacProcessorV3.java | 4 +
.../org/apache/bookkeeper/client/TestFencing.java | 113 +++++++++++++++++++++
9 files changed, 143 insertions(+), 8 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java
index 2b85961cf4..d88673d007 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java
@@ -69,6 +69,8 @@ public abstract class BookieException extends Exception {
return new MetadataStoreException();
case Code.UnknownBookieIdException:
return new UnknownBookieIdException();
+ case Code.LedgerFencedAndDeletedException:
+ return new LedgerFencedAndDeletedException();
case Code.DataUnknownException:
return new DataUnknownException();
default:
@@ -95,6 +97,7 @@ public abstract class BookieException extends Exception {
int CookieExistsException = -109;
int EntryLogMetadataMapException = -110;
int DataUnknownException = -111;
+ int LedgerFencedAndDeletedException = -112;
}
public int getCode() {
@@ -199,6 +202,15 @@ public abstract class BookieException extends Exception {
}
}
+ /**
+ * Signals that a ledger has been fenced in a bookie. No more entries can be appended to that ledger.
+ */
+ public static class LedgerFencedAndDeletedException extends BookieException {
+ public LedgerFencedAndDeletedException() {
+ super(Code.LedgerFencedException);
+ }
+ }
+
/**
* Signals that a ledger's operation has been rejected by an internal component because of the resource saturation.
*/
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java
index b331f12506..ac87c3aed4 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java
@@ -57,7 +57,7 @@ class HandleFactoryImpl implements HandleFactory, LedgerDeletionListener {
if (handle == null) {
if (!journalReplay && recentlyFencedAndDeletedLedgers.getIfPresent(ledgerId) != null) {
- throw BookieException.create(BookieException.Code.LedgerFencedException);
+ throw BookieException.create(BookieException.Code.LedgerFencedAndDeletedException);
}
handle = LedgerDescriptor.create(masterKey, ledgerId, ledgerStorage);
ledgers.putIfAbsent(ledgerId, handle);
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index 80c3a1d9de..7bce38aae2 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
@@ -1165,6 +1165,11 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter {
completion.setOutstanding();
}
} else {
+ try {
+ future.get();
+ } catch (Exception ex) {
+ LOG.warn("Failed to request to the bookie: {}", bookieId, ex);
+ }
nettyOpLogger.registerFailedEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS);
errorOut(key);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java
index 04efd9634b..9c6d672f48 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java
@@ -93,7 +93,7 @@ class ReadEntryProcessor extends PacketProcessorBase<ReadRequest> {
handleReadResultForFenceRead(fenceResult, data, startTimeNanos);
return;
}
- } catch (Bookie.NoLedgerException e) {
+ } catch (Bookie.NoLedgerException | BookieException.LedgerFencedAndDeletedException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Error reading {}", request, e);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java
index 999b8095db..05f9de0ea3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java
@@ -215,9 +215,10 @@ class ReadEntryProcessorV3 extends PacketProcessorBaseV3 {
}
}
return readEntry(readResponse, entryId, startTimeSw);
- } catch (Bookie.NoLedgerException e) {
+ } catch (Bookie.NoLedgerException | BookieException.LedgerFencedAndDeletedException e) {
if (RequestUtils.isFenceRequest(readRequest)) {
- LOG.info("No ledger found reading entry {} when fencing ledger {}", entryId, ledgerId);
+ LOG.info("No ledger found(or it has been deleted) reading entry {} when fencing ledger {}",
+ entryId, ledgerId);
} else if (entryId != BookieProtocol.LAST_ADD_CONFIRMED) {
LOG.info("No ledger found while reading entry: {} from ledger: {}", entryId, ledgerId);
} else if (LOG.isDebugEnabled()) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java
index d611ab963a..665e2d3a1c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java
@@ -90,8 +90,8 @@ class WriteEntryProcessor extends PacketProcessorBase<ParsedAddRequest> implemen
} catch (IOException e) {
LOG.error("Error writing {}", request, e);
rc = BookieProtocol.EIO;
- } catch (BookieException.LedgerFencedException lfe) {
- LOG.warn("Write attempt on fenced ledger {} by client {}", request.getLedgerId(),
+ } catch (BookieException.LedgerFencedException | BookieException.LedgerFencedAndDeletedException lfe) {
+ LOG.warn("Write attempt on fenced/deleted ledger {} by client {}", request.getLedgerId(),
requestHandler.ctx().channel().remoteAddress());
rc = BookieProtocol.EFENCED;
} catch (BookieException e) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java
index 36aff7ad92..79d9f41d5c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java
@@ -136,8 +136,8 @@ class WriteEntryProcessorV3 extends PacketProcessorBaseV3 {
logger.error("Error writing entry:{} to ledger:{}",
entryId, ledgerId, e);
status = StatusCode.EIO;
- } catch (BookieException.LedgerFencedException e) {
- logger.error("Ledger fenced while writing entry:{} to ledger:{}",
+ } catch (BookieException.LedgerFencedException | BookieException.LedgerFencedAndDeletedException e) {
+ logger.error("Ledger fenced/deleted while writing entry:{} to ledger:{}",
entryId, ledgerId, e);
status = StatusCode.EFENCED;
} catch (BookieException e) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java
index 293cea3bb0..81d1bc6605 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java
@@ -105,6 +105,10 @@ class WriteLacProcessorV3 extends PacketProcessorBaseV3 implements Runnable {
requestProcessor.bookie.setExplicitLac(Unpooled.wrappedBuffer(lacToAdd),
writeCallback, requestHandler, masterKey);
status = StatusCode.EOK;
+ } catch (BookieException.LedgerFencedAndDeletedException e) {
+ logger.error("Error saving lac {} for ledger:{}, which has been deleted",
+ lac, ledgerId, e);
+ status = StatusCode.ENOLEDGER;
} catch (IOException e) {
logger.error("Error saving lac {} for ledger:{}",
lac, ledgerId, e);
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java
index 77382b4ebd..9f7a81450b 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java
@@ -24,6 +24,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.UnpooledByteBufAllocator;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import lombok.extern.slf4j.Slf4j;
@@ -33,10 +36,16 @@ import org.apache.bookkeeper.bookie.LedgerStorage;
import org.apache.bookkeeper.bookie.SortedLedgerStorage;
import org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.api.WriteFlag;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.net.BookieId;
+import org.apache.bookkeeper.proto.BookieClientImpl;
+import org.apache.bookkeeper.proto.BookieProtocol;
+import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks;
+import org.apache.bookkeeper.proto.PerChannelBookieClient;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.bookkeeper.test.TestStatsProvider;
+import org.apache.bookkeeper.util.ByteBufList;
import org.awaitility.reflect.WhiteboxImpl;
import org.junit.Test;
import org.slf4j.Logger;
@@ -152,6 +161,110 @@ public class TestFencing extends BookKeeperClusterTestCase {
}
}
+ @Test(timeout = 3000 * 1000)
+ public void testConcurrentFenceAndDeleteLedger() throws Exception {
+ LedgerHandle writeLedger;
+ writeLedger = bkc.createLedger(digestType, "password".getBytes());
+
+ String tmp = "BookKeeper is cool!";
+ long lac = 0;
+ for (int i = 0; i < 10; i++) {
+ long entryId = writeLedger.addEntry(tmp.getBytes());
+ LOG.info("entryId: {}", entryId);
+ lac = entryId;
+ }
+
+ // Fence and delete.
+ final BookieId bookieId = writeLedger.getLedgerMetadata().getEnsembleAt(0).get(0);
+ ClientConfiguration clientConfiguration2 = newClientConfiguration();
+ clientConfiguration2.setUseV2WireProtocol(true);
+ ClientConfiguration clientConfiguration3 = newClientConfiguration();
+ BookKeeperTestClient bkcV2 = new BookKeeperTestClient(clientConfiguration2, new TestStatsProvider());
+ LedgerHandle writeLedgerV2 = bkcV2.createLedger(digestType, "password".getBytes());
+ BookKeeperTestClient bkcV3 = new BookKeeperTestClient(clientConfiguration3, new TestStatsProvider());
+ LedgerHandle writeLedgerV3 = bkcV3.createLedger(digestType, "password".getBytes());
+ ReadOnlyLedgerHandle readLedgerV2 =
+ (ReadOnlyLedgerHandle) bkcV2.openLedger(writeLedger.getId(), digestType, "password".getBytes());
+ ReadOnlyLedgerHandle readLedgerV3 =
+ (ReadOnlyLedgerHandle) bkcV3.openLedger(writeLedger.getId(), digestType, "password".getBytes());
+ BookieClientImpl bookieClientV2 = (BookieClientImpl) readLedgerV2.clientCtx.getBookieClient();
+ BookieClientImpl bookieClientV3 = (BookieClientImpl) readLedgerV3.clientCtx.getBookieClient();
+ // Trigger opening connection.
+ CompletableFuture<Integer> obtainV2 = new CompletableFuture<>();
+ bookieClientV2.lookupClient(bookieId).obtain(
+ new BookkeeperInternalCallbacks.GenericCallback<PerChannelBookieClient>() {
+ @Override
+ public void operationComplete(int rc, PerChannelBookieClient result) {
+ obtainV2.complete(rc);
+ }
+ }, writeLedger.getId());
+ assertEquals(obtainV2.get().intValue(), BKException.Code.OK);
+ CompletableFuture<Integer> obtainV3 = new CompletableFuture<>();
+ bookieClientV3.lookupClient(bookieId).obtain(
+ new BookkeeperInternalCallbacks.GenericCallback<PerChannelBookieClient>() {
+ @Override
+ public void operationComplete(int rc, PerChannelBookieClient result) {
+ obtainV3.complete(rc);
+ }
+ }, writeLedger.getId());
+ assertEquals(obtainV3.get().intValue(), BKException.Code.OK);
+ bkcV3.deleteLedger(readLedgerV3.ledgerId);
+
+ // Waiting for GC.
+ for (ServerTester server : servers) {
+ triggerGC(server.getServer().getBookie());
+ }
+
+ // Verify: read requests with V2 protocol will receive a NoSuchLedgerException.
+ final byte readEntryFlagFencing = 1;
+ CompletableFuture<Integer> readResV2 = new CompletableFuture<>();
+ bookieClientV2.readEntry(bookieId,
+ writeLedger.getId(), 0, (rc, ledgerId, entryId1, buffer, ctx) -> {
+ readResV2.complete(rc);
+ }, null, readEntryFlagFencing, readLedgerV2.ledgerKey);
+ assertEquals(BKException.Code.NoSuchLedgerExistsException, readResV2.get().intValue());
+ // Verify: read requests with V3 protocol will receive a NoSuchLedgerException.
+ CompletableFuture<Integer> readResV3 = new CompletableFuture<>();
+ bookieClientV3.readEntry(bookieId,
+ writeLedger.getId(), 0, (rc, ledgerId, entryId1, buffer, ctx) -> {
+ readResV3.complete(rc);
+ }, null, readEntryFlagFencing, readLedgerV3.ledgerKey);
+ assertEquals(BKException.Code.NoSuchLedgerExistsException, readResV3.get().intValue());
+ // Verify: add requests with V2 protocol will receive a NoSuchLedgerException.
+ log.info("Try to add the next entry: {}:{}", writeLedger.getId(), lac + 1);
+ final ByteBuf dataV2 = UnpooledByteBufAllocator.DEFAULT.heapBuffer();
+ // Combine add request, and rewrite ledgerId of the request.
+ dataV2.writeByte(1);
+ final ByteBuf toSendV2 = (ByteBuf) writeLedgerV2.macManager.computeDigestAndPackageForSending(
+ lac + 1, lac, 1, dataV2, writeLedger.ledgerKey, BookieProtocol.FLAG_NONE);
+ toSendV2.setLong(28, writeLedger.getId());
+ CompletableFuture<Integer> addResV2 = new CompletableFuture<>();
+ bookieClientV2.addEntry(bookieId, writeLedger.getId(), writeLedger.ledgerKey, lac + 1, toSendV2,
+ (rc, ledgerId, entryId1, addr, ctx) -> {
+ addResV2.complete(rc);
+ }, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
+ assertEquals(BKException.Code.LedgerFencedException, addResV2.get().intValue());
+ // Verify: read requests with V3 protocol will receive a NoSuchLedgerException.
+ final ByteBuf dataV3 = UnpooledByteBufAllocator.DEFAULT.heapBuffer();
+ dataV3.writeByte(1);
+ // Combine add request, and rewrite ledgerId of the request.
+ final ByteBufList toSendV3 = (ByteBufList) writeLedgerV3.macManager.computeDigestAndPackageForSending(
+ lac + 1, lac, 1, dataV3, writeLedger.ledgerKey, BookieProtocol.FLAG_NONE);
+ toSendV3.getBuffer(0).setLong(0, writeLedger.getId());
+ CompletableFuture<Integer> addResV3 = new CompletableFuture<>();
+ bookieClientV3.addEntry(bookieId, writeLedger.getId(), writeLedger.ledgerKey, lac + 1, toSendV3,
+ (rc, ledgerId, entryId1, addr, ctx) -> {
+ addResV3.complete(rc);
+ }, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
+ assertEquals(BKException.Code.LedgerFencedException, addResV3.get().intValue());
+
+ // cleanup.
+ writeLedgerV2.close();
+ writeLedgerV3.close();
+ bkcV2.close();
+ bkcV3.close();
+ }
+
private static int threadCount = 0;
class LedgerOpenThread extends Thread {