You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by yo...@apache.org on 2023/02/16 11:15:54 UTC
[bookkeeper] branch branch-4.15 updated (07c38c7513 -> f3eadd6885)
This is an automated email from the ASF dual-hosted git repository.
yong pushed a change to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from 07c38c7513 Expose registrationClient in DefaultBookieAddressResolver (#3724)
new c74c2592de Revert "InterleavedLedgerStorage: safe release ByteBuf (#3674)"
new 4543493a00 Revert "[refactor][bookkeeper] Refactor ByteBuf release method in DefaultEntryLogger (#3673)"
new 5fdef33217 Bump jcommander from 1.78 to 1.82 (#3476)
new 63a90c75bd Adjust log stack output (#3496)
new d03f6a25a7 ledgerFragment check and results keep order (#3504)
new e7848a5f1f LedgerHandle: do not complete metadata operation on the ZookKeeper/Metadata callback thread (#3516)
new e05f0e0bd5 Show content of map (#3538)
new 8f673e5064 Using a separate thread pool to execute openWithMetadata (#3548)
new 0336c3b35e Use LinkedHashMap instead of HashMap to fix flaky tests due non-deterministic order (#3551)
new ed55d4184e Avoid extra buffer to prepend frame size (#3560)
new fbe0e5f08f [configuration] fix typo in bk_server.conf (#3574)
new 716a4e522e Correct link class name (#3594)
new e32e327a54 update doc about flushInterval config (#3601)
new 7614cd6bfd BookieImpl remove wait until journal quits (#3603)
new a7436e6152 Apply recycle logic during add entry creation but ledger close to LedgerHandleAdv (#3621)
new 494f9e9499 Fix GetBookieInfo failed event stats (#3622)
new c0ac377484 [log] Add journal file path that caused failure in multi-journal config (#3623)
new ba473740de Include bkperf into bk all package (#3632)
new fba90caa49 Upgrade docker image version to fix CVEs (#3640)
new a95020ef04 Fix issue where checkAllLedgers could get stuck when read throttling is enabled (#3655)
new 773105e187 Check client if closed when complete callback. (#3661)
new 3a684190f4 [fix][server]fix memory leak when operating ledger metadata (#3662)
new 84ecb39301 Exit bookkeeper shell correctly even if fails to run for some reason (#3663)
new 42504da9d1 Fix RegionAwareEnsemblePlacementPolicy update rack info problem. (#3666)
new 63157d5001 [fix][server]fix memory leak when closeRecovered,failed on clearing newEnsemblesFromRecovery (#3672)
new 5df5d56e97 Make `jvm_memory_direct_bytes_used` metrics compatible with jdk8. (#3677)
new 858637b599 Show result of isFenced in log (#3678)
new a8df522f02 Not wrap IOException twice form checkpoint (#3683)
new 540bc14c04 [improve][doc] improve bk_server.conf docs (#3715)
new a6a407487a Speed up the rebuildinding of RocksDB index (#3458)
new f92171f109 recycle LongPairWrapper (#3728)
new fcff85bebf Fix memory leak when the Bookie is in read-only mode. (#3746)
new f3eadd6885 Fix QueueEntry recycle problem. (#3747)
The 33 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bookkeeper-dist/all/pom.xml | 6 +
.../src/main/resources/LICENSE-all.bin.txt | 6 +-
.../src/main/resources/LICENSE-bkctl.bin.txt | 4 +-
.../src/main/resources/LICENSE-server.bin.txt | 4 +-
.../src/main/resources/NOTICE-all.bin.txt | 2 +-
.../src/main/resources/NOTICE-bkctl.bin.txt | 2 +-
.../src/main/resources/NOTICE-server.bin.txt | 2 +-
.../org/apache/bookkeeper/bookie/BookieImpl.java | 30 +---
.../org/apache/bookkeeper/bookie/BookieShell.java | 78 ++++++-----
.../apache/bookkeeper/bookie/EntryLogMetadata.java | 4 +-
.../org/apache/bookkeeper/bookie/EntryLogger.java | 11 +-
.../bookie/InterleavedLedgerStorage.java | 4 +-
.../java/org/apache/bookkeeper/bookie/Journal.java | 10 +-
.../storage/ldb/LocationsIndexRebuildOp.java | 25 +++-
.../ldb/SingleDirectoryDbLedgerStorage.java | 18 +--
.../bookkeeper/bookie/storage/ldb/WriteCache.java | 5 +-
.../apache/bookkeeper/client/LedgerChecker.java | 5 +-
.../apache/bookkeeper/client/LedgerCreateOp.java | 12 +-
.../org/apache/bookkeeper/client/LedgerHandle.java | 24 +++-
.../apache/bookkeeper/client/LedgerHandleAdv.java | 2 +
.../org/apache/bookkeeper/client/LedgerOpenOp.java | 25 ++--
.../RackawareEnsemblePlacementPolicyImpl.java | 4 +-
.../bookkeeper/client/ReadOnlyLedgerHandle.java | 13 +-
.../client/RegionAwareEnsemblePlacementPolicy.java | 85 ++++++++++--
.../bookkeeper/meta/CleanupLedgerManager.java | 15 +-
.../bookkeeper/meta/HierarchicalLedgerManager.java | 11 ++
.../org/apache/bookkeeper/meta/LedgerManager.java | 27 ++--
.../apache/bookkeeper/proto/BookieNettyServer.java | 6 +-
.../bookkeeper/proto/BookieProtoEncoding.java | 44 ++++--
.../apache/bookkeeper/proto/BookieProtocol.java | 9 +-
.../bookkeeper/proto/GetBookieInfoProcessorV3.java | 6 +-
.../bookkeeper/proto/PacketProcessorBase.java | 4 +-
.../bookkeeper/proto/PerChannelBookieClient.java | 6 +-
.../org/apache/bookkeeper/replication/Auditor.java | 39 ++++--
.../cli/commands/bookie/ListLedgersCommand.java | 4 +-
.../org/apache/bookkeeper/util/ByteBufList.java | 24 +---
.../org/apache/bookkeeper/bookie/EntryLogTest.java | 3 +
.../bookie/storage/ldb/WriteCacheTest.java | 24 +++-
.../bookkeeper/client/BookieWriteLedgerTest.java | 6 +-
.../apache/bookkeeper/client/MockBookKeeper.java | 17 ++-
.../apache/bookkeeper/client/MockLedgerHandle.java | 5 +
.../TestRegionAwareEnsemblePlacementPolicy.java | 154 +++++++++++++++++++++
.../bookkeeper/meta/CleanupLedgerManagerTest.java | 81 +++++++++++
.../bookkeeper/proto/BookieProtoEncodingTest.java | 6 +-
.../proto/GetBookieInfoProcessorV3Test.java | 101 ++++++++++++++
.../replication/AuditorReplicasCheckTest.java | 12 +-
.../apache/bookkeeper/test/ReadOnlyBookieTest.java | 2 +-
.../prometheus-metrics-provider/pom.xml | 6 +-
.../prometheus/PrometheusMetricsProvider.java | 33 ++++-
.../prometheus/TestPrometheusMetricsProvider.java | 7 +-
conf/bk_server.conf | 10 +-
dev/docker/Dockerfile | 2 +-
dev/release/Dockerfile | 2 +-
pom.xml | 2 +-
site3/website/docs/reference/config.md | 2 +-
55 files changed, 804 insertions(+), 247 deletions(-)
create mode 100644 bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/CleanupLedgerManagerTest.java
create mode 100644 bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3Test.java
[bookkeeper] 09/33: Use LinkedHashMap instead of HashMap to fix flaky tests due non-deterministic order (#3551)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 0336c3b35e31ec7600b22a3443c4316ffe7e677e
Author: Aditya Kumbhar <31...@users.noreply.github.com>
AuthorDate: Sun Nov 13 19:24:16 2022 -0600
Use LinkedHashMap instead of HashMap to fix flaky tests due non-deterministic order (#3551)
### Motivation
AuditorReplicasCheckTest has the following tests that are flaky, detected using the [NonDex](https://github.com/TestingResearchIllinois/NonDex) tool.
testReplicasCheckForLedgersFoundHavingLessThanAQReplicasOfAnEntry()
testReplicasCheckForLedgersFoundHavingLessThanWQReplicasOfAnEntry()
testReplicasCheckForLedgersFoundHavingNoReplica()
testReplicasCheckForLedgersWithEmptySegments()
The tests fail due to non-deterministic ordering in the HashMap `segmentEnsembles`
The following check fails in the `newEnsembleEntry()` method since it assumes the ordering of `segmentEnsembles` to be deterministic:
```
checkArgument(ensembles.isEmpty() || firstEntry > ensembles.lastKey(),
"New entry must have a first entry greater than any existing ensemble key");
```
Fixes #3550
Command to reproduce:
```
mvn -pl bookkeeper-server edu.illinois:nondex-maven-plugin:1.1.2:nondex -Dtest=org.apache.bookkeeper.replication.AuditorReplicasCheckTest
```
### Changes
Changed the datatype of `segmentEnsembles` from HashMap to [LinkedHashMap](https://docs.oracle.com/javase/8/docs/api/java/util/LinkedHashMap.html) as it has a deterministic iteration order, while [HashMap](https://docs.oracle.com/javase/8/docs/api/java/util/HashMap.html) does not guarantee the order of its elements.
The tests pass with NonDex after making this change.
(cherry picked from commit f8b9e1886d1f6a806150e450cc525933f6c9063f)
---
.../bookkeeper/replication/AuditorReplicasCheckTest.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java
index dbd6c806f1..00cb2e82d4 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java
@@ -27,7 +27,7 @@ import java.net.URI;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@@ -300,7 +300,7 @@ public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase {
* BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will
* return BookieHandleNotAvailableException.
*/
- Map<Long, List<BookieId>> segmentEnsembles = new HashMap<Long, List<BookieId>>();
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
segmentEnsembles.put(0L, bookieAddresses);
long ledgerId = 1L;
createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
@@ -398,7 +398,7 @@ public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase {
* Empty one for all of the bookies, so this ledger would be counted in
* ledgersFoundHavingNoReplicaOfAnEntry .
*/
- Map<Long, List<BookieId>> segmentEnsembles = new HashMap<Long, List<BookieId>>();
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
segmentEnsembles.put(0L, bookieAddresses);
long ledgerId = 1L;
createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles,
@@ -528,7 +528,7 @@ public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase {
* would be counted towards
* ledgersFoundHavingLessThanAQReplicasOfAnEntry.
*/
- Map<Long, List<BookieId>> segmentEnsembles = new HashMap<Long, List<BookieId>>();
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
int ensembleSize = 4;
int writeQuorumSize = 3;
int ackQuorumSize = 2;
@@ -681,7 +681,7 @@ public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase {
* for this ledger a copy of entry 3, so this ledger would be counted
* towards ledgersFoundHavingLessThanWQReplicasOfAnEntry.
*/
- Map<Long, List<BookieId>> segmentEnsembles = new HashMap<Long, List<BookieId>>();
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
int ensembleSize = 4;
int writeQuorumSize = 3;
int ackQuorumSize = 2;
@@ -836,7 +836,7 @@ public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase {
* numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry
* /WQReplicasOfAnEntry.
*/
- Map<Long, List<BookieId>> segmentEnsembles = new HashMap<Long, List<BookieId>>();
+ Map<Long, List<BookieId>> segmentEnsembles = new LinkedHashMap<Long, List<BookieId>>();
int ensembleSize = 4;
int writeQuorumSize = 3;
int ackQuorumSize = 2;
[bookkeeper] 12/33: Correct link class name (#3594)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 716a4e522e463f742a72b728da92e366be6ee870
Author: gaozhangmin <ga...@qq.com>
AuthorDate: Tue Nov 1 05:01:37 2022 +0800
Correct link class name (#3594)
Co-authored-by: gavingaozhangmin <ga...@didiglobal.com>
(cherry picked from commit 9efd606d1cf1a2f4573606335490548f59b083c2)
---
.../org/apache/bookkeeper/meta/LedgerManager.java | 27 +++++++++++++---------
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
index 56c447d538..d1227e30eb 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
@@ -49,9 +49,10 @@ public interface LedgerManager extends Closeable {
* Metadata provided when creating the new ledger
* @return Future which, when completed returns the metadata of the newly created ledger.
* Completed with an exception:<ul>
- * <li>{@link BKException.BKLedgerExistException} if given ledger id exist</li>
- * <li>{@link BKException.BKZKException}/{@link BKException.BKMetaStoreException} for other issues</li>
- * </ul>
+ * <li>{@link org.apache.bookkeeper.client.BKException.BKLedgerExistException} if given ledger id exist</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.ZKException}
+ * /{@link org.apache.bookkeeper.client.BKException.BKMetadataSerializationException} for other issues</li>
+ * </ul>
*/
CompletableFuture<Versioned<LedgerMetadata>> createLedgerMetadata(long ledgerId, LedgerMetadata metadata);
@@ -64,10 +65,12 @@ public interface LedgerManager extends Closeable {
* Ledger metadata version
* @return Future which, when completed, denotes that the ledger metadata has been removed.
* Completed with an exception:<ul>
- * <li>{@link BKException.BKMetadataVersionException} if version doesn't match</li>
- * <li>{@link BKException.BKNoSuchLedgerExistsOnMetadataServerException} if ledger not exist</li>
- * <li>{@link BKException.ZKException} for other issues</li>
- * </ul>
+ * <li>{@link org.apache.bookkeeper.client.BKException.BKMetadataVersionException}
+ * if version doesn't match</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException}
+ * if ledger not exist</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.ZKException} for other issues</li>
+ * </ul>
*/
CompletableFuture<Void> removeLedgerMetadata(long ledgerId, Version version);
@@ -78,8 +81,9 @@ public interface LedgerManager extends Closeable {
* Ledger Id
* @return Future which, when completed, contains the requested versioned metadata.
* Completed with an exception::<ul>
- * <li>{@link BKException.BKNoSuchLedgerExistsOnMetadataServerException} if ledger not exist</li>
- * <li>{@link BKException.ZKException} for other issues</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException}
+ * if ledger not exist</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.ZKException} for other issues</li>
* </ul>
*/
CompletableFuture<Versioned<LedgerMetadata>> readLedgerMetadata(long ledgerId);
@@ -95,8 +99,9 @@ public interface LedgerManager extends Closeable {
* The version of the metadata we expect to be overwriting.
* @return Future which, when completed, contains the newly written metadata.
* Comleted with an exceptione:<ul>
- * <li>{@link BKException.BKMetadataVersionException} if version in metadata doesn't match</li>
- * <li>{@link BKException.ZKException} for other issue</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.BKMetadataVersionException}
+ * if version in metadata doesn't match</li>
+ * <li>{@link org.apache.bookkeeper.client.BKException.ZKException} for other issue</li>
* </ul>
*/
CompletableFuture<Versioned<LedgerMetadata>> writeLedgerMetadata(long ledgerId, LedgerMetadata metadata,
[bookkeeper] 18/33: Include bkperf into bk all package (#3632)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit ba473740dea96136662937aeb1407a8d6b3e7b39
Author: Hang Chen <ch...@apache.org>
AuthorDate: Sun Nov 13 01:32:43 2022 +0800
Include bkperf into bk all package (#3632)
(cherry picked from commit a9d40822afd48e5e84988e0a41b6140923fb304d)
---
bookkeeper-dist/all/pom.xml | 6 ++++++
bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 2 ++
2 files changed, 8 insertions(+)
diff --git a/bookkeeper-dist/all/pom.xml b/bookkeeper-dist/all/pom.xml
index b9a82bd0e4..24b5a8fa0c 100644
--- a/bookkeeper-dist/all/pom.xml
+++ b/bookkeeper-dist/all/pom.xml
@@ -103,6 +103,12 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-perf</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<!-- needed by ZooKeeper server -->
<groupId>org.xerial.snappy</groupId>
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index 52501f29c2..b2d2196353 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -313,6 +313,7 @@ Apache Software License, Version 2.
- lib/org.conscrypt-conscrypt-openjdk-uber-2.5.1.jar [49]
- lib/org.xerial.snappy-snappy-java-1.1.7.7.jar [50]
- lib/io.reactivex.rxjava3-rxjava-3.0.1.jar [51]
+- lib/org.hdrhistogram-HdrHistogram-2.1.10.jar [52]
[1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0
[2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3
@@ -362,6 +363,7 @@ Apache Software License, Version 2.
[49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.1
[50] Source available at https://github.com/google/snappy/releases/tag/1.1.7.7
[51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1
+[52] Source available at https://github.com/HdrHistogram/HdrHistogram/tree/HdrHistogram-2.1.10
------------------------------------------------------------------------------------
lib/io.netty-netty-codec-4.1.75.Final.jar bundles some 3rd party dependencies
[bookkeeper] 25/33: [fix][server]fix memory leak when closeRecovered,failed on clearing newEnsemblesFromRecovery (#3672)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 63157d500129de33f86edc0fb2d81c1502336138
Author: StevenLuMT <ls...@126.com>
AuthorDate: Wed Dec 7 17:30:17 2022 +0800
[fix][server]fix memory leak when closeRecovered,failed on clearing newEnsemblesFromRecovery (#3672)
Co-authored-by: lushiji <lu...@didiglobal.com>
(cherry picked from commit a19d849a370d0b3eed73bb90f329b47f00730146)
---
.../org/apache/bookkeeper/client/ReadOnlyLedgerHandle.java | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ReadOnlyLedgerHandle.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ReadOnlyLedgerHandle.java
index 1d21fc9377..0f6f4d67aa 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ReadOnlyLedgerHandle.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ReadOnlyLedgerHandle.java
@@ -330,11 +330,14 @@ class ReadOnlyLedgerHandle extends LedgerHandle implements LedgerMetadataListene
return builder.withClosedState().withLastEntryId(lac).withLength(len).build();
},
this::setLedgerMetadata).run();
- f.thenRun(() -> {
- synchronized (metadataLock) {
- newEnsemblesFromRecovery.clear();
- }
- });
+ f.whenComplete((result, exception) -> {
+ synchronized (metadataLock) {
+ newEnsemblesFromRecovery.clear();
+ }
+ if (exception != null) {
+ LOG.error("When closeRecovered,failed on clearing newEnsemblesFromRecovery.", exception);
+ }
+ });
return f;
}
[bookkeeper] 16/33: Fix GetBookieInfo failed event stats (#3622)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 494f9e949930614031f5b4197694961a39c3d57b
Author: wenbingshen <ol...@gmail.com>
AuthorDate: Wed Nov 9 09:09:26 2022 +0800
Fix GetBookieInfo failed event stats (#3622)
### Motivation
In `GetBookieInfoProcessorV3`, even if throw `IOException` and get DiskSpace failed, it will still `registerSuccessfulEvent`, which is wrong, we should `registerFailedEvent`.
### Changes
1. registerFailedEvent when get DiskSpace failed.
2. add unit test.
(cherry picked from commit 74e9ef4d732622fba6cf023cc26ce9a07e066af0)
---
.../bookkeeper/proto/GetBookieInfoProcessorV3.java | 6 +-
.../proto/GetBookieInfoProcessorV3Test.java | 101 +++++++++++++++++++++
2 files changed, 105 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3.java
index fe315f62ca..9978e6c448 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3.java
@@ -74,14 +74,16 @@ public class GetBookieInfoProcessorV3 extends PacketProcessorBaseV3 implements R
getBookieInfoResponse.setTotalDiskCapacity(totalDiskSpace);
}
LOG.debug("FreeDiskSpace info is " + freeDiskSpace + " totalDiskSpace is: " + totalDiskSpace);
+ requestProcessor.getRequestStats().getGetBookieInfoStats()
+ .registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
} catch (IOException e) {
status = StatusCode.EIO;
LOG.error("IOException while getting freespace/totalspace", e);
+ requestProcessor.getRequestStats().getGetBookieInfoStats()
+ .registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
}
getBookieInfoResponse.setStatus(status);
- requestProcessor.getRequestStats().getGetBookieInfoStats()
- .registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
return getBookieInfoResponse.build();
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3Test.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3Test.java
new file mode 100644
index 0000000000..e294a55ccf
--- /dev/null
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/GetBookieInfoProcessorV3Test.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.proto;
+
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import io.netty.channel.Channel;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test {@link GetBookieInfoProcessorV3}.
+ */
+public class GetBookieInfoProcessorV3Test {
+
+ private Channel channel;
+ private BookieRequestProcessor requestProcessor;
+ private Bookie bookie;
+ private RequestStats requestStats;
+ private OpStatsLogger getBookieInfoStats;
+ private OpStatsLogger channelWriteStats;
+ private OpStatsLogger getBookieInfoRequestStats;
+
+ @Before
+ public void setup() {
+ getBookieInfoStats = mock(OpStatsLogger.class);
+ channelWriteStats = mock(OpStatsLogger.class);
+ getBookieInfoRequestStats = mock(OpStatsLogger.class);
+ requestStats = mock(RequestStats.class);
+ requestProcessor = mock(BookieRequestProcessor.class);
+ bookie = mock(Bookie.class);
+ when(requestProcessor.getBookie()).thenReturn(bookie);
+ channel = mock(Channel.class);
+ when(channel.isOpen()).thenReturn(true);
+ when(channel.isActive()).thenReturn(true);
+ when(requestProcessor.getRequestStats()).thenReturn(requestStats);
+ when(requestProcessor.getRequestStats().getGetBookieInfoStats())
+ .thenReturn(getBookieInfoStats);
+ when(requestProcessor.getRequestStats().getChannelWriteStats())
+ .thenReturn(channelWriteStats);
+ when(requestProcessor.getRequestStats().getGetBookieInfoRequestStats())
+ .thenReturn(getBookieInfoRequestStats);
+ }
+
+ @Test
+ public void testGetBookieInfoProcessorStats() throws IOException {
+ final BookkeeperProtocol.BKPacketHeader.Builder headerBuilder =
+ BookkeeperProtocol.BKPacketHeader.newBuilder()
+ .setVersion(BookkeeperProtocol.ProtocolVersion.VERSION_THREE)
+ .setOperation(BookkeeperProtocol.OperationType.GET_BOOKIE_INFO)
+ .setTxnId(0);
+
+ final BookkeeperProtocol.GetBookieInfoRequest.Builder getBookieInfoBuilder =
+ BookkeeperProtocol.GetBookieInfoRequest.newBuilder()
+ .setRequested(BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE);
+
+ final BookkeeperProtocol.Request getBookieInfoRequest = BookkeeperProtocol.Request.newBuilder()
+ .setHeader(headerBuilder)
+ .setGetBookieInfoRequest(getBookieInfoBuilder)
+ .build();
+
+ GetBookieInfoProcessorV3 getBookieInfo = new GetBookieInfoProcessorV3(
+ getBookieInfoRequest, channel, requestProcessor);
+ getBookieInfo.run();
+
+ // get BookieInfo succeeded.
+ verify(getBookieInfoStats, times(1))
+ .registerSuccessfulEvent(anyLong(), eq(TimeUnit.NANOSECONDS));
+
+ // get BookieInfo failed.
+ when(requestProcessor.getBookie().getTotalFreeSpace()).thenThrow(new IOException("test for failed."));
+ getBookieInfo.run();
+ verify(getBookieInfoStats, times(1))
+ .registerFailedEvent(anyLong(), eq(TimeUnit.NANOSECONDS));
+ }
+}
\ No newline at end of file
[bookkeeper] 10/33: Avoid extra buffer to prepend frame size (#3560)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit ed55d4184ee949e2a56459ac4aacf05410d3ac43
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 31 21:05:42 2022 -0700
Avoid extra buffer to prepend frame size (#3560)
* Avoid extra buffer to prepend frame size
* Fixed checkstyle
* Fixed touch methods on ReadResponse
* Fixed frame size for protobuf requests
* Removed unwanted changes
* Fixed AuthResponse in v2 protocol
* Fixed test
(cherry picked from commit 02be9d2ee28cd7c4a1895fc8494db96381484a20)
---
.../apache/bookkeeper/proto/BookieNettyServer.java | 6 +---
.../bookkeeper/proto/BookieProtoEncoding.java | 42 ++++++++++++++++------
.../apache/bookkeeper/proto/BookieProtocol.java | 9 +++--
.../bookkeeper/proto/PerChannelBookieClient.java | 6 ++--
.../org/apache/bookkeeper/util/ByteBufList.java | 24 +------------
.../bookkeeper/proto/BookieProtoEncodingTest.java | 6 ++--
6 files changed, 44 insertions(+), 49 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
index b301bd91ba..7b70a12f3a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
@@ -49,7 +49,6 @@ import io.netty.channel.local.LocalServerChannel;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
-import io.netty.handler.codec.LengthFieldPrepender;
import io.netty.handler.flush.FlushConsolidationHandler;
import io.netty.handler.ssl.SslHandler;
import io.netty.util.concurrent.DefaultThreadFactory;
@@ -332,11 +331,9 @@ class BookieNettyServer {
pipeline.addLast("consolidation", new FlushConsolidationHandler(1024, true));
- // For ByteBufList, skip the usual LengthFieldPrepender and have the encoder itself to add it
- pipeline.addLast("bytebufList", ByteBufList.ENCODER_WITH_SIZE);
+ pipeline.addLast("bytebufList", ByteBufList.ENCODER);
pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.RequestDecoder(registry));
pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.ResponseEncoder(registry));
@@ -401,7 +398,6 @@ class BookieNettyServer {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.RequestDecoder(registry));
pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.ResponseEncoder(registry));
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
index 50fdd2f81c..860fb6635f 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
@@ -109,12 +109,17 @@ public class BookieProtoEncoding {
BookieProtocol.Request r = (BookieProtocol.Request) msg;
if (r instanceof BookieProtocol.AddRequest) {
BookieProtocol.AddRequest ar = (BookieProtocol.AddRequest) r;
- int totalHeaderSize = 4 // for the header
- + BookieProtocol.MASTER_KEY_LENGTH; // for the master key
- ByteBuf buf = allocator.buffer(totalHeaderSize);
+ ByteBufList data = ar.getData();
+
+ int totalHeaderSize = 4 // for the request header
+ + BookieProtocol.MASTER_KEY_LENGTH; // for the master key
+
+ int totalPayloadSize = totalHeaderSize + data.readableBytes();
+ ByteBuf buf = allocator.buffer(totalHeaderSize + 4 /* frame size */);
+ buf.writeInt(totalPayloadSize); // Frame header
buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(), r.getOpCode(), r.getFlags()));
buf.writeBytes(r.getMasterKey(), 0, BookieProtocol.MASTER_KEY_LENGTH);
- ByteBufList data = ar.getData();
+
ar.recycle();
data.prepend(buf);
return data;
@@ -126,7 +131,8 @@ public class BookieProtoEncoding {
totalHeaderSize += BookieProtocol.MASTER_KEY_LENGTH;
}
- ByteBuf buf = allocator.buffer(totalHeaderSize);
+ ByteBuf buf = allocator.buffer(totalHeaderSize + 4 /* frame size */);
+ buf.writeInt(totalHeaderSize);
buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(), r.getOpCode(), r.getFlags()));
buf.writeLong(r.getLedgerId());
buf.writeLong(r.getEntryId());
@@ -139,7 +145,8 @@ public class BookieProtoEncoding {
BookkeeperProtocol.AuthMessage am = ((BookieProtocol.AuthRequest) r).getAuthMessage();
int totalHeaderSize = 4; // for request type
int totalSize = totalHeaderSize + am.getSerializedSize();
- ByteBuf buf = allocator.buffer(totalSize);
+ ByteBuf buf = allocator.buffer(totalSize + 4 /* frame size */);
+ buf.writeInt(totalSize);
buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(), r.getOpCode(), r.getFlags()));
ByteBufOutputStream bufStream = new ByteBufOutputStream(buf);
am.writeTo(bufStream);
@@ -233,6 +240,8 @@ public class BookieProtoEncoding {
this.extensionRegistry = extensionRegistry;
}
+ private static final int RESPONSE_HEADERS_SIZE = 24;
+
@Override
public Object encode(Object msg, ByteBufAllocator allocator)
throws Exception {
@@ -240,7 +249,8 @@ public class BookieProtoEncoding {
return msg;
}
BookieProtocol.Response r = (BookieProtocol.Response) msg;
- ByteBuf buf = allocator.buffer(24);
+ ByteBuf buf = allocator.buffer(RESPONSE_HEADERS_SIZE + 4 /* frame size */);
+ buf.writerIndex(4); // Leave the placeholder for the frame size
buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(), r.getOpCode(), (short) 0));
try {
@@ -251,19 +261,26 @@ public class BookieProtoEncoding {
BookieProtocol.ReadResponse rr = (BookieProtocol.ReadResponse) r;
if (rr.hasData()) {
+ int frameSize = RESPONSE_HEADERS_SIZE + rr.getData().readableBytes();
+ buf.setInt(0, frameSize);
return ByteBufList.get(buf, rr.getData());
} else {
+ buf.setInt(0, RESPONSE_HEADERS_SIZE); // Frame size
return buf;
}
} else if (msg instanceof BookieProtocol.AddResponse) {
buf.writeInt(r.getErrorCode());
buf.writeLong(r.getLedgerId());
buf.writeLong(r.getEntryId());
+ buf.setInt(0, RESPONSE_HEADERS_SIZE); // Frame size
return buf;
} else if (msg instanceof BookieProtocol.AuthResponse) {
BookkeeperProtocol.AuthMessage am = ((BookieProtocol.AuthResponse) r).getAuthMessage();
- return ByteBufList.get(buf, Unpooled.wrappedBuffer(am.toByteArray()));
+ ByteBuf payload = Unpooled.wrappedBuffer(am.toByteArray());
+ int frameSize = 4 + payload.readableBytes();
+ buf.setInt(0, frameSize);
+ return ByteBufList.get(buf, payload);
} else {
LOG.error("Cannot encode unknown response type {}", msg.getClass().getName());
return msg;
@@ -356,6 +373,8 @@ public class BookieProtoEncoding {
private static ByteBuf serializeProtobuf(MessageLite msg, ByteBufAllocator allocator) {
int size = msg.getSerializedSize();
+ int frameSize = size + 4;
+
// Protobuf serialization is the last step of the netty pipeline. We used to allocate
// a heap buffer while serializing and pass it down to netty library.
// In AbstractChannel#filterOutboundMessage(), netty copies that data to a direct buffer if
@@ -363,17 +382,18 @@ public class BookieProtoEncoding {
// Allocating a direct buffer reducing unncessary CPU cycles for buffer copies in BK client
// and also helps alleviate pressure off the GC, since there is less memory churn.
// Bookies aren't usually CPU bound. This change improves READ_ENTRY code paths by a small factor as well.
- ByteBuf buf = allocator.directBuffer(size, size);
+ ByteBuf buf = allocator.directBuffer(frameSize, frameSize);
+ buf.writeInt(size);
try {
- msg.writeTo(CodedOutputStream.newInstance(buf.nioBuffer(buf.readerIndex(), size)));
+ msg.writeTo(CodedOutputStream.newInstance(buf.nioBuffer(buf.writerIndex(), size)));
} catch (IOException e) {
// This is in-memory serialization, should not fail
throw new RuntimeException(e);
}
// Advance writer idx
- buf.writerIndex(buf.capacity());
+ buf.writerIndex(frameSize);
return buf;
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
index 965f3fd1bb..c3b09b856d 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
@@ -471,7 +471,8 @@ public interface BookieProtocol {
@Override
public ReferenceCounted retain() {
- return data.retain();
+ data.retain();
+ return this;
}
@Override
@@ -481,12 +482,14 @@ public interface BookieProtocol {
@Override
public ReferenceCounted touch() {
- return data.touch();
+ data.touch();
+ return this;
}
@Override
public ReferenceCounted touch(Object hint) {
- return data.touch(hint);
+ data.touch(hint);
+ return this;
}
@Override
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index 1b1092bd39..2cda0d243b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
@@ -52,7 +52,6 @@ import io.netty.channel.unix.Errors.NativeIoException;
import io.netty.handler.codec.CorruptedFrameException;
import io.netty.handler.codec.DecoderException;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
-import io.netty.handler.codec.LengthFieldPrepender;
import io.netty.handler.codec.TooLongFrameException;
import io.netty.handler.flush.FlushConsolidationHandler;
import io.netty.handler.ssl.SslHandler;
@@ -584,10 +583,9 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter {
protected void initChannel(Channel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast("consolidation", new FlushConsolidationHandler(1024, true));
- pipeline.addLast("bytebufList", ByteBufList.ENCODER_WITH_SIZE);
+ pipeline.addLast("bytebufList", ByteBufList.ENCODER);
pipeline.addLast("lengthbasedframedecoder",
new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.RequestEncoder(extRegistry));
pipeline.addLast(
"bookieProtoDecoder",
@@ -1273,7 +1271,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter {
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
exceptionCounter.inc();
if (cause instanceof CorruptedFrameException || cause instanceof TooLongFrameException) {
- LOG.error("Corrupted frame received from bookie: {}", ctx.channel().remoteAddress());
+ LOG.error("Corrupted frame received from bookie: {}", ctx.channel());
ctx.close();
return;
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ByteBufList.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ByteBufList.java
index d136ff0c58..f74652bb10 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ByteBufList.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ByteBufList.java
@@ -44,8 +44,6 @@ import java.util.ArrayList;
* will need to be encoded on the channel. There are 2 utility encoders:
* <ul>
* <li>{@link #ENCODER}: regular encode that will write all the buffers in the {@link ByteBufList} on the channel</li>
- * <li>{@link #ENCODER_WITH_SIZE}: similar to the previous one, but also prepend a 4 bytes size header, once, carrying
- * the size of the readable bytes across all the buffers contained in the {@link ByteBufList}</li>
* </ul>
*
* <p>Example:
@@ -307,13 +305,7 @@ public class ByteBufList extends AbstractReferenceCounted {
/**
* Encoder for the {@link ByteBufList} that doesn't prepend any size header.
*/
- public static final Encoder ENCODER = new Encoder(false);
-
- /**
- * Encoder for the {@link ByteBufList} that will prepend a 4 byte header with the size of the whole
- * {@link ByteBufList} readable bytes.
- */
- public static final Encoder ENCODER_WITH_SIZE = new Encoder(true);
+ public static final Encoder ENCODER = new Encoder();
/**
* {@link ByteBufList} encoder.
@@ -321,26 +313,12 @@ public class ByteBufList extends AbstractReferenceCounted {
@Sharable
public static class Encoder extends ChannelOutboundHandlerAdapter {
- private final boolean prependSize;
-
- public Encoder(boolean prependSize) {
- this.prependSize = prependSize;
- }
-
@Override
public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
if (msg instanceof ByteBufList) {
ByteBufList b = (ByteBufList) msg;
try {
- if (prependSize) {
- // Prepend the frame size before writing the buffer list, so that we only have 1 single size
- // header
- ByteBuf sizeBuffer = ctx.alloc().directBuffer(4, 4);
- sizeBuffer.writeInt(b.readableBytes());
- ctx.write(sizeBuffer, ctx.voidPromise());
- }
-
// Write each buffer individually on the socket. The retain() here is needed to preserve the fact
// that ByteBuf are automatically released after a write. If the ByteBufPair ref count is increased
// and it gets written multiple times, the individual buffers refcount should be reflected as well.
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/BookieProtoEncodingTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/BookieProtoEncodingTest.java
index 537c9fcfc7..bba26f37a2 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/BookieProtoEncodingTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/BookieProtoEncodingTest.java
@@ -102,9 +102,9 @@ public class BookieProtoEncodingTest {
}
assertEquals(0, outList.size());
- v3Decoder.channelRead(
- ctx,
- v3Encoder.encode(v3Resp, UnpooledByteBufAllocator.DEFAULT));
+ ByteBuf serWithFrameSize = (ByteBuf) v3Encoder.encode(v3Resp, UnpooledByteBufAllocator.DEFAULT);
+ ByteBuf ser = serWithFrameSize.slice(4, serWithFrameSize.readableBytes() - 4);
+ v3Decoder.channelRead(ctx, ser);
assertEquals(1, outList.size());
}
[bookkeeper] 29/33: [improve][doc] improve bk_server.conf docs (#3715)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 540bc14c04fe89f7131445256102b6a909b2fb5c
Author: Paul Gier <pa...@datastax.com>
AuthorDate: Thu Jan 5 02:29:17 2023 -0600
[improve][doc] improve bk_server.conf docs (#3715)
* [improve][doc] improve bk_server.conf docs
Add default value of allowMultipleDirsUnderSameDiskPartition
Signed-off-by: Paul Gier <pa...@datastax.com>
* [improve][doc] change example conf to match default
Signed-off-by: Paul Gier <pa...@datastax.com>
Signed-off-by: Paul Gier <pa...@datastax.com>
(cherry picked from commit 05378442f776febf9f8ef49f10779a9e39438bd5)
---
conf/bk_server.conf | 5 +++--
site3/website/docs/reference/config.md | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index a241a31a2b..2c09c2c2f9 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -41,8 +41,9 @@
bookiePort=3181
# Configure the bookie to allow/disallow multiple ledger/index/journal directories
-# in the same filesystem disk partition
-# allowMultipleDirsUnderSameDiskPartition=false
+# in the same filesystem disk partition.
+# Defaults to true.
+# allowMultipleDirsUnderSameDiskPartition=true
# Set the network interface that the bookie should listen on.
# If not set, the bookie will listen on all interfaces.
diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md
index 64272fab99..c4e7d3adcb 100644
--- a/site3/website/docs/reference/config.md
+++ b/site3/website/docs/reference/config.md
@@ -12,7 +12,7 @@ The table below lists parameters that you can set to configure bookies. All conf
| Parameter | Description | Default
| --------- | ----------- | ------- |
| bookiePort | The port that the bookie server listens on. | 3181 |
-| allowMultipleDirsUnderSameDiskPartition | Configure the bookie to allow/disallow multiple ledger/index/journal directories in the same filesystem disk partition | |
+| allowMultipleDirsUnderSameDiskPartition | Configure the bookie to allow/disallow multiple ledger/index/journal directories in the same filesystem disk partition. | true |
| listeningInterface | The network interface that the bookie should listen on. If not set, the bookie will listen on all interfaces. | eth0 |
| advertisedAddress | Configure a specific hostname or IP address that the bookie should use to advertise itself to<br />clients. If not set, bookie will advertised its own IP address or hostname, depending on the<br />`listeningInterface` and `useHostNameAsBookieID` settings.<br /> | eth0 |
| allowLoopback | Whether the bookie is allowed to use a loopback interface as its primary<br />interface (the interface it uses to establish its identity). By default, loopback interfaces are *not* allowed as the primary interface.<br /><br />Using a loopback interface as the primary interface usually indicates a configuration error. It's fairly common in some VPS setups, for example, to not configure a hostname or to have the hostname resolve to 127.0.0.1. If this is the case, then all [...]
[bookkeeper] 27/33: Show result of isFenced in log (#3678)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 858637b5993a7c08a252cb4a2364c7c95ff5121b
Author: gaozhangmin <ga...@qq.com>
AuthorDate: Wed Dec 7 10:20:05 2022 +0800
Show result of isFenced in log (#3678)
Log `isFenced`, output to log file
(cherry picked from commit 394ff12dc180c351227f28ce31031e74b96f4352)
---
.../bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
index 1777e2e1c1..fbc57823ba 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
@@ -342,12 +342,12 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
@Override
public boolean isFenced(long ledgerId) throws IOException, BookieException {
+ boolean isFenced = ledgerIndex.get(ledgerId).getFenced();
+
if (log.isDebugEnabled()) {
- log.debug("isFenced. ledger: {}", ledgerId);
+ log.debug("ledger: {}, isFenced: {}.", ledgerId, isFenced);
}
- boolean isFenced = ledgerIndex.get(ledgerId).getFenced();
-
// Only a negative result while in limbo equates to unknown
if (!isFenced) {
throwIfLimbo(ledgerId);
[bookkeeper] 24/33: Fix RegionAwareEnsemblePlacementPolicy update rack info problem. (#3666)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 42504da9d10352eec3760bf16d5ce756d43e7031
Author: Yan Zhao <ho...@apache.org>
AuthorDate: Wed Dec 7 15:50:09 2022 +0800
Fix RegionAwareEnsemblePlacementPolicy update rack info problem. (#3666)
(cherry picked from commit 4574ba02333308ad648c9b78963a381ad83ea564)
---
.../client/RegionAwareEnsemblePlacementPolicy.java | 85 ++++++++++--
.../TestRegionAwareEnsemblePlacementPolicy.java | 154 +++++++++++++++++++++
2 files changed, 225 insertions(+), 14 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java
index 383260dc80..b7a7f0e48e 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java
@@ -20,6 +20,7 @@ package org.apache.bookkeeper.client;
import io.netty.util.HashedWheelTimer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -36,6 +37,7 @@ import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.net.BookieNode;
import org.apache.bookkeeper.net.DNSToSwitchMapping;
import org.apache.bookkeeper.net.NetworkTopology;
+import org.apache.bookkeeper.net.NetworkTopologyImpl;
import org.apache.bookkeeper.net.Node;
import org.apache.bookkeeper.net.NodeBase;
import org.apache.bookkeeper.proto.BookieAddressResolver;
@@ -85,30 +87,34 @@ public class RegionAwareEnsemblePlacementPolicy extends RackawareEnsemblePlaceme
address2Region = new ConcurrentHashMap<BookieId, String>();
}
+ protected String getLocalRegion(BookieNode node) {
+ if (null == node || null == node.getAddr()) {
+ return UNKNOWN_REGION;
+ }
+ return getRegion(node.getAddr());
+ }
+
protected String getRegion(BookieId addr) {
String region = address2Region.get(addr);
if (null == region) {
- String networkLocation = resolveNetworkLocation(addr);
- if (NetworkTopology.DEFAULT_REGION_AND_RACK.equals(networkLocation)) {
- region = UNKNOWN_REGION;
- } else {
- String[] parts = networkLocation.split(NodeBase.PATH_SEPARATOR_STR);
- if (parts.length <= 1) {
- region = UNKNOWN_REGION;
- } else {
- region = parts[1];
- }
- }
+ region = parseBookieRegion(addr);
address2Region.putIfAbsent(addr, region);
}
return region;
}
- protected String getLocalRegion(BookieNode node) {
- if (null == node || null == node.getAddr()) {
+ protected String parseBookieRegion(BookieId addr) {
+ String networkLocation = resolveNetworkLocation(addr);
+ if (NetworkTopology.DEFAULT_REGION_AND_RACK.equals(networkLocation)) {
return UNKNOWN_REGION;
+ } else {
+ String[] parts = networkLocation.split(NodeBase.PATH_SEPARATOR_STR);
+ if (parts.length <= 1) {
+ return UNKNOWN_REGION;
+ } else {
+ return parts[1];
+ }
}
- return getRegion(node.getAddr());
}
@Override
@@ -163,6 +169,57 @@ public class RegionAwareEnsemblePlacementPolicy extends RackawareEnsemblePlaceme
}
}
+ @Override
+ public void onBookieRackChange(List<BookieId> bookieAddressList) {
+ rwLock.writeLock().lock();
+ try {
+ bookieAddressList.forEach(bookieAddress -> {
+ try {
+ BookieNode node = knownBookies.get(bookieAddress);
+ if (node != null) {
+ // refresh the rack info if its a known bookie
+ BookieNode newNode = createBookieNode(bookieAddress);
+ if (!newNode.getNetworkLocation().equals(node.getNetworkLocation())) {
+ topology.remove(node);
+ topology.add(newNode);
+ knownBookies.put(bookieAddress, newNode);
+ historyBookies.put(bookieAddress, newNode);
+ }
+ //Handle per region placement policy.
+ String oldRegion = getRegion(bookieAddress);
+ String newRegion = parseBookieRegion(newNode.getAddr());
+ if (oldRegion.equals(newRegion)) {
+ TopologyAwareEnsemblePlacementPolicy regionPlacement = perRegionPlacement.get(oldRegion);
+ regionPlacement.onBookieRackChange(Collections.singletonList(bookieAddress));
+ } else {
+ address2Region.put(bookieAddress, newRegion);
+ TopologyAwareEnsemblePlacementPolicy oldRegionPlacement = perRegionPlacement.get(oldRegion);
+ oldRegionPlacement.handleBookiesThatLeft(Collections.singleton(bookieAddress));
+ TopologyAwareEnsemblePlacementPolicy newRegionPlacement = perRegionPlacement.get(
+ newRegion);
+ if (newRegionPlacement == null) {
+ newRegionPlacement = new RackawareEnsemblePlacementPolicy()
+ .initialize(dnsResolver, timer, this.reorderReadsRandom,
+ this.stabilizePeriodSeconds, this.reorderThresholdPendingRequests,
+ this.isWeighted, this.maxWeightMultiple,
+ this.minNumRacksPerWriteQuorum, this.enforceMinNumRacksPerWriteQuorum,
+ this.ignoreLocalNodeInPlacementPolicy, statsLogger,
+ bookieAddressResolver)
+ .withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
+ perRegionPlacement.put(newRegion, newRegionPlacement);
+ }
+ newRegionPlacement.handleBookiesThatJoined(Collections.singleton(bookieAddress));
+ }
+ }
+ } catch (IllegalArgumentException | NetworkTopologyImpl.InvalidTopologyException e) {
+ LOG.error("Failed to update bookie rack info: {} ", bookieAddress, e);
+ }
+ });
+ } finally {
+ rwLock.writeLock().unlock();
+ }
+ }
+
@Override
public RegionAwareEnsemblePlacementPolicy initialize(ClientConfiguration conf,
Optional<DNSToSwitchMapping> optionalDnsResolver,
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java
index fc4e02cae9..7fe5b83db4 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java
@@ -28,6 +28,7 @@ import static org.apache.bookkeeper.client.RegionAwareEnsemblePlacementPolicy.RE
import static org.apache.bookkeeper.client.RoundRobinDistributionSchedule.writeSetFromValues;
import static org.apache.bookkeeper.feature.SettableFeatureProvider.DISABLE_ALL;
+import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.netty.util.HashedWheelTimer;
import java.net.InetAddress;
@@ -1528,6 +1529,7 @@ public class TestRegionAwareEnsemblePlacementPolicy extends TestCase {
assertEquals(ensemble.get(reoderSet.get(7)), addr4.toBookieId());
}
+ @Test
public void testNewEnsembleSetWithFiveRegions() throws Exception {
repp.uninitalize();
repp = new RegionAwareEnsemblePlacementPolicy();
@@ -1573,6 +1575,7 @@ public class TestRegionAwareEnsemblePlacementPolicy extends TestCase {
}
}
+ @Test
public void testRegionsWithDiskWeight() throws Exception {
repp.uninitalize();
repp = new RegionAwareEnsemblePlacementPolicy();
@@ -1607,4 +1610,155 @@ public class TestRegionAwareEnsemblePlacementPolicy extends TestCase {
assertEquals(3, ensemble.size());
}
+
+ @Test
+ public void testNotifyRackChangeWithOldRegion() throws Exception {
+ BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181);
+ BookieSocketAddress addr2 = new BookieSocketAddress("127.0.1.2", 3181);
+ BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.3", 3181);
+ BookieSocketAddress addr4 = new BookieSocketAddress("127.0.1.4", 3181);
+
+ // update dns mapping
+ StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/rack-1");
+ StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/rack-1");
+ StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region2/rack-1");
+ StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region2/rack-1");
+
+ // Update cluster
+ Set<BookieId> addrs = Sets.newHashSet(addr1.toBookieId(),
+ addr2.toBookieId(), addr3.toBookieId(), addr4.toBookieId());
+ repp.onClusterChanged(addrs, new HashSet<>());
+
+ assertEquals(4, repp.knownBookies.size());
+ assertEquals("/region1/rack-1", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-1", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals(2, repp.perRegionPlacement.size());
+ TopologyAwareEnsemblePlacementPolicy region1Placement = repp.perRegionPlacement.get("region1");
+ assertEquals(2, region1Placement.knownBookies.keySet().size());
+ assertEquals("/region1/rack-1", region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-1", region1Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+
+ TopologyAwareEnsemblePlacementPolicy region2Placement = repp.perRegionPlacement.get("region2");
+ assertEquals(2, region2Placement.knownBookies.keySet().size());
+ assertEquals("/region2/rack-1", region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", region2Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals("region1", repp.address2Region.get(addr1.toBookieId()));
+ assertEquals("region1", repp.address2Region.get(addr2.toBookieId()));
+ assertEquals("region2", repp.address2Region.get(addr3.toBookieId()));
+ assertEquals("region2", repp.address2Region.get(addr4.toBookieId()));
+
+ // Update the rack.
+ // change addr2 rack info. /region1/rack-1 -> /region1/rack-2.
+ // change addr4 rack info. /region2/rack-1 -> /region1/rack-2
+ List<BookieSocketAddress> bookieAddressList = new ArrayList<>();
+ List<String> rackList = new ArrayList<>();
+ bookieAddressList.add(addr2);
+ rackList.add("/region1/rack-2");
+ bookieAddressList.add(addr4);
+ rackList.add("/region1/rack-2");
+ StaticDNSResolver.changeRack(bookieAddressList, rackList);
+
+ assertEquals(4, repp.knownBookies.size());
+ assertEquals("/region1/rack-1", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-2", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-2", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals(2, repp.perRegionPlacement.size());
+ region1Placement = repp.perRegionPlacement.get("region1");
+ assertEquals(3, region1Placement.knownBookies.keySet().size());
+ assertEquals("/region1/rack-1", region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-2", region1Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-2", region1Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ region2Placement = repp.perRegionPlacement.get("region2");
+ assertEquals(1, region2Placement.knownBookies.keySet().size());
+ assertEquals("/region2/rack-1", region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+
+ assertEquals("region1", repp.address2Region.get(addr1.toBookieId()));
+ assertEquals("region1", repp.address2Region.get(addr2.toBookieId()));
+ assertEquals("region2", repp.address2Region.get(addr3.toBookieId()));
+ assertEquals("region1", repp.address2Region.get(addr4.toBookieId()));
+ }
+
+ @Test
+ public void testNotifyRackChangeWithNewRegion() throws Exception {
+ BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181);
+ BookieSocketAddress addr2 = new BookieSocketAddress("127.0.1.2", 3181);
+ BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.3", 3181);
+ BookieSocketAddress addr4 = new BookieSocketAddress("127.0.1.4", 3181);
+
+ // update dns mapping
+ StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/rack-1");
+ StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/rack-1");
+ StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region2/rack-1");
+ StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region2/rack-1");
+
+ // Update cluster
+ Set<BookieId> addrs = Sets.newHashSet(addr1.toBookieId(),
+ addr2.toBookieId(), addr3.toBookieId(), addr4.toBookieId());
+ repp.onClusterChanged(addrs, new HashSet<>());
+
+ assertEquals(4, repp.knownBookies.size());
+ assertEquals("/region1/rack-1", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-1", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals(2, repp.perRegionPlacement.size());
+ TopologyAwareEnsemblePlacementPolicy region1Placement = repp.perRegionPlacement.get("region1");
+ assertEquals(2, region1Placement.knownBookies.keySet().size());
+ assertEquals("/region1/rack-1", region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region1/rack-1", region1Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+
+ TopologyAwareEnsemblePlacementPolicy region2Placement = repp.perRegionPlacement.get("region2");
+ assertEquals(2, region2Placement.knownBookies.keySet().size());
+ assertEquals("/region2/rack-1", region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", region2Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals("region1", repp.address2Region.get(addr1.toBookieId()));
+ assertEquals("region1", repp.address2Region.get(addr2.toBookieId()));
+ assertEquals("region2", repp.address2Region.get(addr3.toBookieId()));
+ assertEquals("region2", repp.address2Region.get(addr4.toBookieId()));
+
+ // Update the rack.
+ // change addr2 rack info. /region1/rack-1 -> /region3/rack-1.
+ // change addr4 rack info. /region2/rack-1 -> /region3/rack-1
+ List<BookieSocketAddress> bookieAddressList = new ArrayList<>();
+ List<String> rackList = new ArrayList<>();
+ bookieAddressList.add(addr2);
+ rackList.add("/region3/rack-1");
+ bookieAddressList.add(addr4);
+ rackList.add("/region3/rack-1");
+ StaticDNSResolver.changeRack(bookieAddressList, rackList);
+
+ assertEquals(4, repp.knownBookies.size());
+ assertEquals("/region1/rack-1", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+ assertEquals("/region3/rack-1", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+ assertEquals("/region2/rack-1", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+ assertEquals("/region3/rack-1", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals(3, repp.perRegionPlacement.size());
+ region1Placement = repp.perRegionPlacement.get("region1");
+ assertEquals(1, region1Placement.knownBookies.keySet().size());
+ assertEquals("/region1/rack-1", region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
+
+ region2Placement = repp.perRegionPlacement.get("region2");
+ assertEquals(1, region2Placement.knownBookies.keySet().size());
+ assertEquals("/region2/rack-1", region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
+
+ TopologyAwareEnsemblePlacementPolicy region3Placement = repp.perRegionPlacement.get("region3");
+ assertEquals(2, region3Placement.knownBookies.keySet().size());
+ assertEquals("/region3/rack-1", region3Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
+ assertEquals("/region3/rack-1", region3Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
+
+ assertEquals("region1", repp.address2Region.get(addr1.toBookieId()));
+ assertEquals("region3", repp.address2Region.get(addr2.toBookieId()));
+ assertEquals("region2", repp.address2Region.get(addr3.toBookieId()));
+ assertEquals("region3", repp.address2Region.get(addr4.toBookieId()));
+ }
}
[bookkeeper] 06/33: LedgerHandle: do not complete metadata operation on the ZookKeeper/Metadata callback thread (#3516)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit e7848a5f1f231bc5bd92170dce464d96b7ef32a2
Author: Enrico Olivelli <en...@datastax.com>
AuthorDate: Thu Oct 6 14:00:14 2022 +0200
LedgerHandle: do not complete metadata operation on the ZookKeeper/Metadata callback thread (#3516)
(cherry picked from commit 21357087f08b57a530c94591121b9803b5a486e5)
---
.../apache/bookkeeper/client/LedgerCreateOp.java | 12 ++++++++++-
.../org/apache/bookkeeper/client/LedgerHandle.java | 24 +++++++++++++++-------
.../org/apache/bookkeeper/client/LedgerOpenOp.java | 13 +++++++++++-
.../bookkeeper/client/BookieWriteLedgerTest.java | 6 +++---
.../apache/bookkeeper/client/MockBookKeeper.java | 17 ++++++++++++---
.../apache/bookkeeper/client/MockLedgerHandle.java | 5 +++++
6 files changed, 62 insertions(+), 15 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
index 7dfdd16161..959664a8f3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
@@ -47,6 +47,7 @@ import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.util.MathUtils;
+import org.apache.bookkeeper.util.SafeRunnable;
import org.apache.bookkeeper.versioning.Versioned;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -272,7 +273,16 @@ class LedgerCreateOp {
} else {
createOpLogger.registerSuccessfulEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS);
}
- cb.createComplete(rc, lh, ctx);
+ if (lh != null) { // lh is null in case of errors
+ lh.executeOrdered(new SafeRunnable() {
+ @Override
+ public void safeRun() {
+ cb.createComplete(rc, lh, ctx);
+ }
+ });
+ } else {
+ cb.createComplete(rc, null, ctx);
+ }
}
public static class CreateBuilderImpl implements CreateBuilder {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
index 50fc2d6624..1d55bb32ba 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
@@ -539,7 +539,7 @@ public class LedgerHandle implements WriteHandle {
* @param rc
*/
void doAsyncCloseInternal(final CloseCallback cb, final Object ctx, final int rc) {
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, new SafeRunnable() {
+ executeOrdered(new SafeRunnable() {
@Override
public void safeRun() {
final HandleState prevHandleState;
@@ -896,7 +896,7 @@ public class LedgerHandle implements WriteHandle {
if (isHandleWritable()) {
// Ledger handle in read/write mode: submit to OSE for ordered execution.
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, op);
+ executeOrdered(op);
} else {
// Read-only ledger handle: bypass OSE and execute read directly in client thread.
// This avoids a context-switch to OSE thread and thus reduces latency.
@@ -1159,7 +1159,7 @@ public class LedgerHandle implements WriteHandle {
if (wasClosed) {
// make sure the callback is triggered in main worker pool
try {
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, new SafeRunnable() {
+ executeOrdered(new SafeRunnable() {
@Override
public void safeRun() {
LOG.warn("Force() attempted on a closed ledger: {}", ledgerId);
@@ -1179,7 +1179,7 @@ public class LedgerHandle implements WriteHandle {
// early exit: no write has been issued yet
if (pendingAddsSequenceHead == INVALID_ENTRY_ID) {
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, new SafeRunnable() {
+ executeOrdered(new SafeRunnable() {
@Override
public void safeRun() {
FutureUtils.complete(result, null);
@@ -1194,7 +1194,7 @@ public class LedgerHandle implements WriteHandle {
}
try {
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, op);
+ executeOrdered(op);
} catch (RejectedExecutionException e) {
result.completeExceptionally(new BKException.BKInterruptedException());
}
@@ -1328,7 +1328,7 @@ public class LedgerHandle implements WriteHandle {
if (wasClosed) {
// make sure the callback is triggered in main worker pool
try {
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, new SafeRunnable() {
+ executeOrdered(new SafeRunnable() {
@Override
public void safeRun() {
LOG.warn("Attempt to add to closed ledger: {}", ledgerId);
@@ -1363,7 +1363,7 @@ public class LedgerHandle implements WriteHandle {
}
try {
- clientCtx.getMainWorkerPool().executeOrdered(ledgerId, op);
+ executeOrdered(op);
} catch (RejectedExecutionException e) {
op.cb.addCompleteWithLatency(
BookKeeper.getReturnRc(clientCtx.getBookieClient(), BKException.Code.InterruptedException),
@@ -2085,4 +2085,14 @@ public class LedgerHandle implements WriteHandle {
return distributionSchedule.getWriteSet(entryId);
}
}
+
+ /**
+ * Execute the callback in the thread pinned to the ledger.
+ * @param runnable
+ * @throws RejectedExecutionException
+ */
+ void executeOrdered(org.apache.bookkeeper.common.util.SafeRunnable runnable) throws RejectedExecutionException {
+ clientCtx.getMainWorkerPool().executeOrdered(ledgerId, runnable);
+ }
+
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
index 0b2be9e64c..6ba4a11e3b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
@@ -40,6 +40,7 @@ import org.apache.bookkeeper.client.impl.OpenBuilderBase;
import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.util.MathUtils;
import org.apache.bookkeeper.util.OrderedGenericCallback;
+import org.apache.bookkeeper.util.SafeRunnable;
import org.apache.bookkeeper.versioning.Versioned;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -256,7 +257,17 @@ class LedgerOpenOp {
} else {
openOpLogger.registerSuccessfulEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS);
}
- cb.openComplete(rc, lh, ctx);
+
+ if (lh != null) { // lh is null in case of errors
+ lh.executeOrdered(new SafeRunnable() {
+ @Override
+ public void safeRun() {
+ cb.openComplete(rc, lh, ctx);
+ }
+ });
+ } else {
+ cb.openComplete(rc, null, ctx);
+ }
}
static final class OpenBuilderImpl extends OpenBuilderBase {
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
index fb9f83a6aa..ce2d6f9b64 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
@@ -731,7 +731,7 @@ public class BookieWriteLedgerTest extends
.withDigestType(org.apache.bookkeeper.client.api.DigestType.CRC32)
.withPassword(ledgerPassword).makeAdv().withLedgerId(ledgerId)
.execute()
- .thenApply(writer -> { // Add entries to ledger when created
+ .thenCompose(writer -> { // Add entries to ledger when created
LOG.info("Writing stream of {} entries to {}",
numEntriesToWrite, ledgerId);
List<ByteBuf> entries = rng.ints(numEntriesToWrite, 0, maxInt)
@@ -750,8 +750,8 @@ public class BookieWriteLedgerTest extends
ledgerId, entryId, entry.slice().readInt());
lastRequest = writer.writeAsync(entryId, entry);
}
- lastRequest.join();
- return Pair.of(writer, entries);
+ return lastRequest
+ .thenApply(___ -> Pair.of(writer, entries));
});
})
.parallel().map(CompletableFuture::join) // wait for all creations and adds in parallel
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeper.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeper.java
index 75f3d8eeb9..9f5889f3f4 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeper.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeper.java
@@ -40,6 +40,7 @@ import org.apache.bookkeeper.client.api.OpenBuilder;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.client.impl.OpenBuilderBase;
import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.util.SafeRunnable;
import org.apache.zookeeper.ZooKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -107,9 +108,14 @@ public class MockBookKeeper extends BookKeeper {
log.info("Creating ledger {}", id);
MockLedgerHandle lh = new MockLedgerHandle(MockBookKeeper.this, id, digestType, passwd);
ledgers.put(id, lh);
- cb.createComplete(0, lh, ctx);
+ lh.executeOrdered(new SafeRunnable() {
+ @Override
+ public void safeRun() {
+ cb.createComplete(0, lh, ctx);
+ }
+ });
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Error", t);
}
}
});
@@ -164,7 +170,12 @@ public class MockBookKeeper extends BookKeeper {
} else if (!Arrays.equals(lh.passwd, passwd)) {
cb.openComplete(BKException.Code.UnauthorizedAccessException, null, ctx);
} else {
- cb.openComplete(0, lh, ctx);
+ lh.executeOrdered(new SafeRunnable() {
+ @Override
+ public void safeRun() {
+ cb.openComplete(0, lh, ctx);
+ }
+ });
}
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockLedgerHandle.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockLedgerHandle.java
index e2b897d841..feca1e8867 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockLedgerHandle.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockLedgerHandle.java
@@ -92,6 +92,11 @@ public class MockLedgerHandle extends LedgerHandle {
}
+ @Override
+ void executeOrdered(org.apache.bookkeeper.common.util.SafeRunnable runnable) throws RejectedExecutionException {
+ bk.executor.execute(runnable);
+ }
+
@Override
public void asyncReadEntries(final long firstEntry, final long lastEntry, final ReadCallback cb, final Object ctx) {
if (bk.isStopped()) {
[bookkeeper] 30/33: Speed up the rebuildinding of RocksDB index (#3458)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit a6a407487ae4bd64caf25c06e27cfbe7c64c3f5d
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Sep 19 08:59:57 2022 -0700
Speed up the rebuildinding of RocksDB index (#3458)
* Speed up the rebuildinding of RocksDB index
* fix check style
Co-authored-by: chenhang <ch...@apache.org>
(cherry picked from commit 7004d994938e914561d8343a96d9d75ce98b2837)
---
.../bookie/storage/ldb/LocationsIndexRebuildOp.java | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
index 55c5c90c31..e15457ca8e 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
@@ -32,7 +32,8 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.bookkeeper.bookie.BookieImpl;
import org.apache.bookkeeper.bookie.EntryLogger;
import org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner;
@@ -55,6 +56,8 @@ public class LocationsIndexRebuildOp {
this.conf = conf;
}
+ private static final int BATCH_COMMIT_SIZE = 10_000;
+
public void initiate() throws IOException {
LOG.info("Starting locations index rebuilding");
@@ -82,6 +85,8 @@ public class LocationsIndexRebuildOp {
int totalEntryLogs = entryLogs.size();
int completedEntryLogs = 0;
LOG.info("Scanning {} entry logs", totalEntryLogs);
+ AtomicReference<KeyValueStorage.Batch> batch = new AtomicReference<>(newIndex.newBatch());
+ AtomicInteger count = new AtomicInteger();
for (long entryLogId : entryLogs) {
entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {
@@ -100,7 +105,15 @@ public class LocationsIndexRebuildOp {
// Update the ledger index page
LongPairWrapper key = LongPairWrapper.get(ledgerId, entryId);
LongWrapper value = LongWrapper.get(location);
- newIndex.put(key.array, value.array);
+ batch.get().put(key.array, value.array);
+
+ if (count.incrementAndGet() > BATCH_COMMIT_SIZE) {
+ batch.get().flush();
+ batch.get().close();
+
+ batch.set(newIndex.newBatch());
+ count.set(0);
+ }
}
@Override
@@ -112,6 +125,8 @@ public class LocationsIndexRebuildOp {
++completedEntryLogs;
LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs,
totalEntryLogs);
+ batch.get().flush();
+ batch.get().close();
}
newIndex.sync();
[bookkeeper] 07/33: Show content of map (#3538)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit e05f0e0bd548b8302833a905020de1a75958f391
Author: gaozhangmin <ga...@qq.com>
AuthorDate: Wed Oct 19 08:51:44 2022 +0800
Show content of map (#3538)
Fixes #3490
Change ToString method, print the content of map.
(cherry picked from commit 5c6e78f5f477673338df09309b3b1f95c75458dd)
---
.../src/main/java/org/apache/bookkeeper/bookie/EntryLogMetadata.java | 4 ++--
.../src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java | 3 +++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogMetadata.java
index 0445033b52..44cb708119 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogMetadata.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogMetadata.java
@@ -107,8 +107,8 @@ public class EntryLogMetadata {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("{ totalSize = ").append(totalSize).append(", remainingSize = ").append(remainingSize)
- .append(", ledgersMap = ").append(ledgersMap).append(" }");
+ sb.append("{totalSize = ").append(totalSize).append(", remainingSize = ").append(remainingSize)
+ .append(", ledgersMap = ").append(ledgersMap.toString()).append("}");
return sb.toString();
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
index e3627f40ab..4971d31f2a 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/EntryLogTest.java
@@ -192,6 +192,9 @@ public class EntryLogTest {
entryLogger = new EntryLogger(conf, dirsMgr);
EntryLogMetadata meta = entryLogger.getEntryLogMetadata(0L);
+ String metaString = meta.toString();
+ assertEquals(metaString,
+ "{totalSize = 60, remainingSize = 60, ledgersMap = ConcurrentLongLongHashMap{1 => 30, 3 => 30}}");
LOG.info("Extracted Meta From Entry Log {}", meta);
assertTrue(meta.getLedgersMap().containsKey(1L));
assertFalse(meta.getLedgersMap().containsKey(2L));
[bookkeeper] 20/33: Fix issue where checkAllLedgers could get stuck when read throttling is enabled (#3655)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit a95020ef041617113143f322d97bd140560df5a3
Author: Masahiro Sakamoto <ma...@yahoo-corp.jp>
AuthorDate: Thu Feb 9 12:22:30 2023 +0900
Fix issue where checkAllLedgers could get stuck when read throttling is enabled (#3655)
### Motivation
Some time ago, I fixed a bug where check of all ledgers periodically run by the auditor got stuck (cf. https://github.com/apache/bookkeeper/pull/3214). That PR was merged so we cherry-picked it and released it, but `checkAllLedgers` still got stuck when the `inFlightReadEntryNumInLedgerChecker` value was very small.
The cause is that multiple `BookKeeperClientWorker-OrderedExecutor` threads are blocked waiting for the semaphore to be released. This semaphore is released by the `BookKeeperClientWorker-OrderedExecutor` threads, so they should not run the `LedgerChecker#checkLedger` method.
```
"BookKeeperClientWorker-OrderedExecutor-1-0" #60 prio=5 os_prio=0 tid=0x00007f680401d800 nid=0x75e8 waiting on condition [0x00007f684a6e9000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000a2121780> (a java.util.concurrent.Semaphore$NonfairSync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.Semaphore.acquire(Semaphore.java:467)
at org.apache.bookkeeper.client.LedgerChecker.acquirePermit(LedgerChecker.java:163)
at org.apache.bookkeeper.client.LedgerChecker.checkLedger(LedgerChecker.java:431)
at org.apache.bookkeeper.replication.Auditor.lambda$null$4(Auditor.java:1216)
at org.apache.bookkeeper.replication.Auditor$$Lambda$137/1553334572.openComplete(Unknown Source)
at org.apache.bookkeeper.client.LedgerOpenOp.openComplete(LedgerOpenOp.java:232)
at org.apache.bookkeeper.client.LedgerOpenOp$2.readLastConfirmedComplete(LedgerOpenOp.java:218)
at org.apache.bookkeeper.client.LedgerHandle$14.getLacComplete(LedgerHandle.java:1718)
at org.apache.bookkeeper.client.PendingReadLacOp.readLacComplete(PendingReadLacOp.java:154)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadLacCompletion$1.readLacComplete(PerChannelBookieClient.java:1801)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadLacCompletion.handleV3Response(PerChannelBookieClient.java:1840)
at org.apache.bookkeeper.proto.PerChannelBookieClient$3.safeRun(PerChannelBookieClient.java:1473)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:750)
```
### Changes
In the `Auditor` class, have the `LedgerChecker#checkLedger` method run in a different thread from the `BookKeeperClientWorker-OrderedExecutor` threads.
Master Issue: https://github.com/apache/bookkeeper/issues/3070
(cherry picked from commit 4ca4b4e4845ec1a3d5b17c96c99e97a62b5ec6c7)
---
.../org/apache/bookkeeper/replication/Auditor.java | 39 ++++++++++++++++------
1 file changed, 29 insertions(+), 10 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
index 9ddc659fa6..baf3d8b7cb 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
@@ -68,6 +68,7 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
@@ -146,6 +147,7 @@ public class Auditor implements AutoCloseable {
private LedgerManager ledgerManager;
private LedgerUnderreplicationManager ledgerUnderreplicationManager;
private final ScheduledExecutorService executor;
+ private final ExecutorService ledgerCheckerExecutor;
private List<String> knownBookies = new ArrayList<String>();
private final String bookieIdentifier;
private volatile Future<?> auditTask;
@@ -501,6 +503,14 @@ public class Auditor implements AutoCloseable {
return t;
}
});
+ ledgerCheckerExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r, "AuditorBookie-LedgerChecker-" + bookieIdentifier);
+ t.setDaemon(true);
+ return t;
+ }
+ });
}
private void initialize(ServerConfiguration conf, BookKeeper bkc)
@@ -1370,16 +1380,20 @@ public class Auditor implements AutoCloseable {
localAdmin.asyncOpenLedgerNoRecovery(ledgerId, (rc, lh, ctx) -> {
openLedgerNoRecoverySemaphore.release();
if (Code.OK == rc) {
- checker.checkLedger(lh,
- // the ledger handle will be closed after checkLedger is done.
- new ProcessLostFragmentsCb(lh, callback),
- conf.getAuditorLedgerVerificationPercentage());
- // we collect the following stats to get a measure of the
- // distribution of a single ledger within the bk cluster
- // the higher the number of fragments/bookies, the more distributed it is
- numFragmentsPerLedger.registerSuccessfulValue(lh.getNumFragments());
- numBookiesPerLedger.registerSuccessfulValue(lh.getNumBookies());
- numLedgersChecked.inc();
+ // BookKeeperClientWorker-OrderedExecutor threads should not execute LedgerChecker#checkLedger
+ // as this can lead to deadlocks
+ ledgerCheckerExecutor.execute(() -> {
+ checker.checkLedger(lh,
+ // the ledger handle will be closed after checkLedger is done.
+ new ProcessLostFragmentsCb(lh, callback),
+ conf.getAuditorLedgerVerificationPercentage());
+ // we collect the following stats to get a measure of the
+ // distribution of a single ledger within the bk cluster
+ // the higher the number of fragments/bookies, the more distributed it is
+ numFragmentsPerLedger.registerSuccessfulValue(lh.getNumFragments());
+ numBookiesPerLedger.registerSuccessfulValue(lh.getNumBookies());
+ numLedgersChecked.inc();
+ });
} else if (Code.NoSuchLedgerExistsOnMetadataServerException == rc) {
if (LOG.isDebugEnabled()) {
LOG.debug("Ledger {} was deleted before we could check it", ledgerId);
@@ -2116,11 +2130,16 @@ public class Auditor implements AutoCloseable {
public void shutdown() {
LOG.info("Shutting down auditor");
executor.shutdown();
+ ledgerCheckerExecutor.shutdown();
try {
while (!executor.awaitTermination(30, TimeUnit.SECONDS)) {
LOG.warn("Executor not shutting down, interrupting");
executor.shutdownNow();
}
+ while (!ledgerCheckerExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+ LOG.warn("Executor for ledger checker not shutting down, interrupting");
+ ledgerCheckerExecutor.shutdownNow();
+ }
if (ownAdmin) {
admin.close();
}
[bookkeeper] 22/33: [fix][server]fix memory leak when operating ledger metadata (#3662)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 3a684190f493323bf5b28c48ce295197c76af11b
Author: Qiang Huang <qi...@gmail.com>
AuthorDate: Sun Nov 27 21:35:40 2022 +0800
[fix][server]fix memory leak when operating ledger metadata (#3662)
* [fix][server]fix memory leak when operating ledger metadata
(cherry picked from commit 109688c1b460621d636d601580a407b721ca282a)
---
.../bookkeeper/meta/CleanupLedgerManager.java | 15 +++-
.../bookkeeper/meta/CleanupLedgerManagerTest.java | 81 ++++++++++++++++++++++
2 files changed, 94 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
index a3ce432961..8c0cb0bf90 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
@@ -26,7 +26,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.common.concurrent.FutureUtils;
@@ -40,6 +40,7 @@ import org.apache.zookeeper.AsyncCallback;
/**
* A ledger manager that cleans up resources upon closing.
*/
+@Slf4j
public class CleanupLedgerManager implements LedgerManager {
private class CleanupGenericCallback<T> implements GenericCallback<T> {
@@ -113,7 +114,17 @@ public class CleanupLedgerManager implements LedgerManager {
private void recordPromise(CompletableFuture<?> promise) {
futures.add(promise);
- promise.thenRun(() -> futures.remove(promise));
+ promise.whenComplete((result, exception) -> {
+ futures.remove(promise);
+ if (exception != null) {
+ log.error("Failed on operating ledger metadata: {}", BKException.getExceptionCode(exception));
+ }
+ });
+ }
+
+ @VisibleForTesting
+ int getCurrentFuturePromiseSize() {
+ return futures.size();
}
@Override
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/CleanupLedgerManagerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/CleanupLedgerManagerTest.java
new file mode 100644
index 0000000000..fabbdc7f05
--- /dev/null
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/CleanupLedgerManagerTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.meta;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.versioning.Version;
+import org.apache.bookkeeper.versioning.Versioned;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test of {@link CleanupLedgerManager}.
+ */
+public class CleanupLedgerManagerTest {
+
+ protected LedgerManager ledgerManager = null;
+ protected CleanupLedgerManager cleanupLedgerManager = null;
+
+ @Before
+ public void setup() throws Exception {
+ ledgerManager = mock(LedgerManager.class);
+ CompletableFuture<Versioned<LedgerMetadata>> future = new CompletableFuture<>();
+ future.completeExceptionally(new Exception("LedgerNotExistException"));
+ when(ledgerManager.createLedgerMetadata(anyLong(), any())).thenReturn(future);
+ when(ledgerManager.readLedgerMetadata(anyLong())).thenReturn(future);
+ when(ledgerManager.writeLedgerMetadata(anyLong(), any(), any())).thenReturn(
+ future);
+ CompletableFuture<Void> removeFuture = new CompletableFuture<>();
+ removeFuture.completeExceptionally(new Exception("LedgerNotExistException"));
+ when(ledgerManager.removeLedgerMetadata(anyLong(), any())).thenReturn(removeFuture);
+ cleanupLedgerManager = new CleanupLedgerManager(ledgerManager);
+ }
+
+ @Test
+ public void testCreateLedgerMetadataException() throws Exception {
+ cleanupLedgerManager.createLedgerMetadata(anyLong(), any(LedgerMetadata.class));
+ Assert.assertEquals(0, cleanupLedgerManager.getCurrentFuturePromiseSize());
+ }
+
+ @Test
+ public void testReadLedgerMetadataException() throws Exception {
+ cleanupLedgerManager.readLedgerMetadata(anyLong());
+ Assert.assertEquals(0, cleanupLedgerManager.getCurrentFuturePromiseSize());
+ }
+
+ @Test
+ public void testWriteLedgerMetadataException() throws Exception {
+ cleanupLedgerManager.writeLedgerMetadata(anyLong(), any(LedgerMetadata.class), any(Version.class));
+ Assert.assertEquals(0, cleanupLedgerManager.getCurrentFuturePromiseSize());
+ }
+
+ @Test
+ public void testRemoveLedgerMetadataException() throws Exception {
+ cleanupLedgerManager.removeLedgerMetadata(anyLong(), any(Version.class));
+ Assert.assertEquals(0, cleanupLedgerManager.getCurrentFuturePromiseSize());
+ }
+}
[bookkeeper] 33/33: Fix QueueEntry recycle problem. (#3747)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit f3eadd6885d6d13526d555b008a11b20977cf291
Author: Yan Zhao <ho...@apache.org>
AuthorDate: Fri Feb 3 10:34:41 2023 +0800
Fix QueueEntry recycle problem. (#3747)
Descriptions of the changes in this PR:
In the QueueEntry recycle, it only recycles itself to the object pool, but didn't reset some properties.
Like entry, cb, etc. We should reset the filed before recycles itself.
(cherry picked from commit 901f76ce4c4f9f771363424dbb60da4d590ad122)
---
.../src/main/java/org/apache/bookkeeper/bookie/Journal.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index 62c07377d3..6105d00012 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -348,8 +348,8 @@ public class Journal extends BookieCriticalThread implements CheckpointSource {
cbThreadPoolQueueSize.dec();
journalAddEntryStats.registerSuccessfulEvent(MathUtils.elapsedNanos(enqueueTime), TimeUnit.NANOSECONDS);
cb.writeComplete(0, ledgerId, entryId, null, ctx);
- recycle();
callbackTime.add(MathUtils.elapsedNanos(startTime));
+ recycle();
}
private final Handle<QueueEntry> recyclerHandle;
@@ -366,6 +366,14 @@ public class Journal extends BookieCriticalThread implements CheckpointSource {
};
private void recycle() {
+ this.entry = null;
+ this.cb = null;
+ this.ctx = null;
+ this.journalAddEntryStats = null;
+ this.journalCbQueuedLatency = null;
+ this.journalCbQueueSize = null;
+ this.cbThreadPoolQueueSize = null;
+ this.callbackTime = null;
recyclerHandle.recycle(this);
}
}
[bookkeeper] 15/33: Apply recycle logic during add entry creation but ledger close to LedgerHandleAdv (#3621)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit a7436e6152eac85150058e84de11a38b415953ee
Author: zhaohaidao <zh...@hotmail.com>
AuthorDate: Mon Nov 14 09:27:55 2022 +0800
Apply recycle logic during add entry creation but ledger close to LedgerHandleAdv (#3621)
Descriptions of the changes in this PR:
Apply recycle logic during add entry creation but ledger close to LedgerHandleAdv
### Motivation
This pr is a supplement to #3321
If I understand correctly. these recycled logic should be applied to LedgerHandleAdv
(cherry picked from commit 0d2fbe434ec2412a166103ebf7fe9c28ecfc48f6)
---
.../src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
index 1320fc0fd2..a30824b044 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
@@ -249,6 +249,7 @@ public class LedgerHandleAdv extends LedgerHandle implements WriteAdvHandle {
LOG.warn("Attempt to add to closed ledger: {}", ledgerId);
op.cb.addCompleteWithLatency(BKException.Code.LedgerClosedException,
LedgerHandleAdv.this, op.getEntryId(), 0, op.ctx);
+ op.recyclePendAddOpObject();
}
@Override
public String toString() {
@@ -259,6 +260,7 @@ public class LedgerHandleAdv extends LedgerHandle implements WriteAdvHandle {
op.cb.addCompleteWithLatency(BookKeeper.getReturnRc(clientCtx.getBookieClient(),
BKException.Code.InterruptedException),
LedgerHandleAdv.this, op.getEntryId(), 0, op.ctx);
+ op.recyclePendAddOpObject();
}
return;
}
[bookkeeper] 26/33: Make `jvm_memory_direct_bytes_used` metrics compatible with jdk8. (#3677)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 5df5d56e979397fe036baf656a5d312183bbff8e
Author: Yan Zhao <ho...@apache.org>
AuthorDate: Wed Dec 7 15:52:08 2022 +0800
Make `jvm_memory_direct_bytes_used` metrics compatible with jdk8. (#3677)
(cherry picked from commit 5a38080ccfd4be051d5c276283e0f24c21ef938d)
---
.../prometheus-metrics-provider/pom.xml | 6 +++-
.../prometheus/PrometheusMetricsProvider.java | 33 ++++++++++++++++++----
.../prometheus/TestPrometheusMetricsProvider.java | 7 +++--
3 files changed, 37 insertions(+), 9 deletions(-)
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
index a39d48791c..6768847c44 100644
--- a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
@@ -52,7 +52,11 @@
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
index ab81992531..607b16aae6 100644
--- a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java
@@ -34,6 +34,7 @@ import java.io.IOException;
import java.io.Writer;
import java.lang.management.BufferPoolMXBean;
import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.List;
@@ -43,6 +44,8 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.StatsProvider;
import org.apache.bookkeeper.stats.ThreadRegistry;
@@ -130,7 +133,7 @@ public class PrometheusMetricsProvider implements StatsProvider {
registerMetrics(Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() {
@Override
public double get() {
- return poolMxBeanOp.isPresent() ? poolMxBeanOp.get().getMemoryUsed() : Double.NaN;
+ return getDirectMemoryUsage.get();
}
}));
@@ -212,14 +215,34 @@ public class PrometheusMetricsProvider implements StatsProvider {
}
}
-
private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class);
+ /*
+ * Try to get Netty counter of used direct memory. This will be correct, unlike the JVM values.
+ */
+ private static final AtomicLong directMemoryUsage;
private static final Optional<BufferPoolMXBean> poolMxBeanOp;
+ private static final Supplier<Double> getDirectMemoryUsage;
static {
- List<BufferPoolMXBean> platformMXBeans = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class);
- poolMxBeanOp = platformMXBeans.stream()
- .filter(bufferPoolMXBean -> bufferPoolMXBean.getName().equals("direct")).findAny();
+ if (PlatformDependent.useDirectBufferNoCleaner()) {
+ poolMxBeanOp = Optional.empty();
+ AtomicLong tmpDirectMemoryUsage = null;
+ try {
+ Field field = PlatformDependent.class.getDeclaredField("DIRECT_MEMORY_COUNTER");
+ field.setAccessible(true);
+ tmpDirectMemoryUsage = (AtomicLong) field.get(null);
+ } catch (Throwable t) {
+ log.warn("Failed to access netty DIRECT_MEMORY_COUNTER field {}", t.getMessage());
+ }
+ directMemoryUsage = tmpDirectMemoryUsage;
+ getDirectMemoryUsage = () -> directMemoryUsage != null ? directMemoryUsage.get() : Double.NaN;
+ } else {
+ directMemoryUsage = null;
+ List<BufferPoolMXBean> platformMXBeans = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class);
+ poolMxBeanOp = platformMXBeans.stream()
+ .filter(bufferPoolMXBean -> bufferPoolMXBean.getName().equals("direct")).findAny();
+ getDirectMemoryUsage = () -> poolMxBeanOp.isPresent() ? poolMxBeanOp.get().getMemoryUsed() : Double.NaN;
+ }
}
}
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java b/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java
index 999be26cbb..850f8b57b3 100644
--- a/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java
@@ -21,8 +21,9 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
import java.io.StringWriter;
-import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashMap;
@@ -122,7 +123,7 @@ public class TestPrometheusMetricsProvider {
config.setProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_HTTP_ENABLE, true);
config.setProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_HTTP_PORT, 0);
config.setProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_HTTP_ADDRESS, "127.0.0.1");
- ByteBuffer byteBuffer = ByteBuffer.allocateDirect(25);
+ ByteBuf byteBuf = ByteBufAllocator.DEFAULT.directBuffer(25);
PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
try {
provider.start(config);
@@ -145,7 +146,7 @@ public class TestPrometheusMetricsProvider {
Assert.assertNotEquals("Nan", directBytesUsed);
Assert.assertTrue(Double.parseDouble(directBytesUsed) > 25);
// ensure byteBuffer doesn't gc
- byteBuffer.clear();
+ byteBuf.release();
} finally {
provider.stop();
}
[bookkeeper] 23/33: Exit bookkeeper shell correctly even if fails to run for some reason (#3663)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 84ecb39301fc89a7635adae9e684a9af7cd6d809
Author: Yuri Mizushima <yu...@yahoo-corp.jp>
AuthorDate: Fri Nov 25 20:28:08 2022 +0900
Exit bookkeeper shell correctly even if fails to run for some reason (#3663)
* fix: close legacyLM and longLM if call HierarchicalLedgerManager#close
* fix: exit as ok when calling listledgers command and get NoSuchLedgerExistsOnMetadataServerException
* fix: call exit even if get some exception in bookkeeler shell
(cherry picked from commit dd8ba477ee1f7b376869777ecf37463e8d2edec5)
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 78 ++++++++++++----------
.../bookkeeper/meta/HierarchicalLedgerManager.java | 11 +++
.../cli/commands/bookie/ListLedgersCommand.java | 4 +-
3 files changed, 56 insertions(+), 37 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 0eeb0dc601..731a5dd5dc 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -2395,46 +2395,52 @@ public class BookieShell implements Tool {
}
}
- public static void main(String[] argv) throws Exception {
- BookieShell shell = new BookieShell();
-
- // handle some common options for multiple cmds
- Options opts = new Options();
- opts.addOption(CONF_OPT, true, "configuration file");
- opts.addOption(LEDGERID_FORMATTER_OPT, true, "format of ledgerId");
- opts.addOption(ENTRY_FORMATTER_OPT, true, "format of entries");
- BasicParser parser = new BasicParser();
- CommandLine cmdLine = parser.parse(opts, argv, true);
-
- // load configuration
- CompositeConfiguration conf = new CompositeConfiguration();
- if (cmdLine.hasOption(CONF_OPT)) {
- String val = cmdLine.getOptionValue(CONF_OPT);
- conf.addConfiguration(new PropertiesConfiguration(
+ public static void main(String[] argv) {
+ int res = -1;
+ try {
+ BookieShell shell = new BookieShell();
+
+ // handle some common options for multiple cmds
+ Options opts = new Options();
+ opts.addOption(CONF_OPT, true, "configuration file");
+ opts.addOption(LEDGERID_FORMATTER_OPT, true, "format of ledgerId");
+ opts.addOption(ENTRY_FORMATTER_OPT, true, "format of entries");
+ BasicParser parser = new BasicParser();
+ CommandLine cmdLine = parser.parse(opts, argv, true);
+
+ // load configuration
+ CompositeConfiguration conf = new CompositeConfiguration();
+ if (cmdLine.hasOption(CONF_OPT)) {
+ String val = cmdLine.getOptionValue(CONF_OPT);
+ conf.addConfiguration(new PropertiesConfiguration(
new File(val).toURI().toURL()));
- }
- shell.setConf(conf);
+ }
+ shell.setConf(conf);
- // ledgerid format
- if (cmdLine.hasOption(LEDGERID_FORMATTER_OPT)) {
- String val = cmdLine.getOptionValue(LEDGERID_FORMATTER_OPT);
- shell.ledgerIdFormatter = LedgerIdFormatter.newLedgerIdFormatter(val, shell.bkConf);
- } else {
- shell.ledgerIdFormatter = LedgerIdFormatter.newLedgerIdFormatter(shell.bkConf);
- }
- LOG.debug("Using ledgerIdFormatter {}", shell.ledgerIdFormatter.getClass());
+ // ledgerid format
+ if (cmdLine.hasOption(LEDGERID_FORMATTER_OPT)) {
+ String val = cmdLine.getOptionValue(LEDGERID_FORMATTER_OPT);
+ shell.ledgerIdFormatter = LedgerIdFormatter.newLedgerIdFormatter(val, shell.bkConf);
+ } else {
+ shell.ledgerIdFormatter = LedgerIdFormatter.newLedgerIdFormatter(shell.bkConf);
+ }
+ LOG.debug("Using ledgerIdFormatter {}", shell.ledgerIdFormatter.getClass());
- // entry format
- if (cmdLine.hasOption(ENTRY_FORMATTER_OPT)) {
- String val = cmdLine.getOptionValue(ENTRY_FORMATTER_OPT);
- shell.entryFormatter = EntryFormatter.newEntryFormatter(val, shell.bkConf);
- } else {
- shell.entryFormatter = EntryFormatter.newEntryFormatter(shell.bkConf);
- }
- LOG.debug("Using entry formatter {}", shell.entryFormatter.getClass());
+ // entry format
+ if (cmdLine.hasOption(ENTRY_FORMATTER_OPT)) {
+ String val = cmdLine.getOptionValue(ENTRY_FORMATTER_OPT);
+ shell.entryFormatter = EntryFormatter.newEntryFormatter(val, shell.bkConf);
+ } else {
+ shell.entryFormatter = EntryFormatter.newEntryFormatter(shell.bkConf);
+ }
+ LOG.debug("Using entry formatter {}", shell.entryFormatter.getClass());
- int res = shell.run(cmdLine.getArgs());
- System.exit(res);
+ res = shell.run(cmdLine.getArgs());
+ } catch (Throwable e) {
+ LOG.error("Got an exception", e);
+ } finally {
+ System.exit(res);
+ }
}
private synchronized void initEntryLogger() throws IOException {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/HierarchicalLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/HierarchicalLedgerManager.java
index b5fd0f21ae..81ef9e7dff 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/HierarchicalLedgerManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/HierarchicalLedgerManager.java
@@ -123,4 +123,15 @@ class HierarchicalLedgerManager extends AbstractHierarchicalLedgerManager {
protected String getLedgerParentNodeRegex() {
return StringUtils.HIERARCHICAL_LEDGER_PARENT_NODE_REGEX;
}
+
+ @Override
+ public void close() {
+ super.close();
+ if (legacyLM != null) {
+ legacyLM.close();
+ }
+ if (longLM != null) {
+ longLM.close();
+ }
+ }
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/ListLedgersCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/ListLedgersCommand.java
index c743222844..462c5abcbe 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/ListLedgersCommand.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/ListLedgersCommand.java
@@ -157,7 +157,9 @@ public class ListLedgersCommand extends BookieCommand<ListLedgersFlags> {
}
cb.processResult(BKException.Code.OK, null, null);
} else if (BKException.getExceptionCode(exception)
- == BKException.Code.NoSuchLedgerExistsException) {
+ == BKException.Code.NoSuchLedgerExistsException
+ || BKException.getExceptionCode(exception)
+ == BKException.Code.NoSuchLedgerExistsOnMetadataServerException) {
cb.processResult(BKException.Code.OK, null, null);
} else {
LOG.error("Unable to read the ledger: {} information", ledgerId);
[bookkeeper] 08/33: Using a separate thread pool to execute openWithMetadata (#3548)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 8f673e50647fcff1b9f796461d86eef862deb917
Author: gaozhangmin <ga...@qq.com>
AuthorDate: Thu Nov 3 17:04:43 2022 +0800
Using a separate thread pool to execute openWithMetadata (#3548)
(cherry picked from commit 5724bc8434a4d6903487221d39713f39a4bcb6b4)
---
.../main/java/org/apache/bookkeeper/client/LedgerOpenOp.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
index 6ba4a11e3b..847da0d89c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
@@ -113,12 +113,10 @@ class LedgerOpenOp {
* Asynchronously read the ledger metadata node.
*/
bk.getLedgerManager().readLedgerMetadata(ledgerId)
- .whenComplete((metadata, exception) -> {
- if (exception != null) {
- openComplete(BKException.getExceptionCode(exception), null);
- } else {
- openWithMetadata(metadata);
- }
+ .thenAcceptAsync(this::openWithMetadata, bk.getScheduler().chooseThread(ledgerId))
+ .exceptionally(exception -> {
+ openComplete(BKException.getExceptionCode(exception), null);
+ return null;
});
}
[bookkeeper] 05/33: ledgerFragment check and results keep order (#3504)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit d03f6a25a72753de47dfac270a42d9a2d7c72771
Author: wenbingshen <ol...@gmail.com>
AuthorDate: Wed Sep 28 12:18:12 2022 +0800
ledgerFragment check and results keep order (#3504)
### Motivation
When `LedgerChecker` checks Ledger's fragments, it does not keep the order, and the results of the check are not kept in order, which works correctly most of the time.
I'm not sure if we should keep the order of checks and final results, but I observed a flaky-test like this:
https://github.com/apache/bookkeeper/actions/runs/3124965464/jobs/5068848112
I have printed some verbose logs as follows:
testShouldGetTwoFrgamentsIfTwoBookiesFailedInSameEnsemble In the case of success and failure:
### When the test execution is successful:
2022-09-26T15:00:13,790 - INFO - [Time-limited test:LedgerChecker@373] - checkLedger nowing.
2022-09-26T15:00:13,792 - INFO - [Time-limited test:LedgerChecker@398] - LedgerCheck fragments before[Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50466, 127.0.0.1:50474, 127.0.0.1:50476], Closed: true)]
2022-09-26T15:00:13,793 - INFO - [Time-limited test:LedgerChecker@430] - not execute check fragments. curEnsemble [127.0.0.1:50466, 127.0.0.1:50486, 127.0.0.1:50476], curEntryId 1
**2022-09-26T15:00:13,809 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker$1@440] - check2Fragments [Fragment(LedgerID: 0, **FirstEntryID: 0[0]**, LastKnownEntryID: 0[0], Host: [127.0.0.1:50466, 127.0.0.1:50474, 127.0.0.1:50476], Closed: true), Fragment(LedgerID: 0, **FirstEntryID: 1[1]**, LastKnownEntryID: 1[1], Host: [127.0.0.1:50466, 127.0.0.1:50476, 127.0.0.1:50486], Closed: false)]**
2022-09-26T15:00:13,810 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker@479] - Checking fragment Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50466, 127.0.0.1:50474, 127.0.0.1:50476], Closed: true)
2022-09-26T15:00:13,813 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker$FullLedgerCallback@360] - operationComplete Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50466, 127.0.0.1:50474], Closed: true), -8
2022-09-26T15:00:13,813 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker@479] - Checking fragment Fragment(LedgerID: 0, FirstEntryID: 1[1], LastKnownEntryID: 1[1], Host: [127.0.0.1:50466, 127.0.0.1:50476, 127.0.0.1:50486], Closed: false)
2022-09-26T15:00:13,817 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker$FullLedgerCallback@360] - operationComplete Fragment(LedgerID: 0, FirstEntryID: 1[-1], LastKnownEntryID: 1[-1], Host: [127.0.0.1:50466], Closed: false), -8
**2022-09-26T15:00:13,817 - INFO - [Time-limited test:TestLedgerChecker@534] - waitAndGetResult [Fragment(LedgerID: 0, **FirstEntryID: 0[0]**, LastKnownEntryID: 0[0], Host: [127.0.0.1:50466, 127.0.0.1:50474], Closed: true), Fragment(LedgerID: 0, **FirstEntryID: 1[-1]**, LastKnownEntryID: 1[-1], Host: [127.0.0.1:50466], Closed: false)]**
2022-09-26T15:00:13,818 - INFO - [Time-limited test:TestLedgerChecker@201] - other Ensemble {0=[127.0.0.1:50466, 127.0.0.1:50474, 127.0.0.1:50476], 1=[127.0.0.1:50466, 127.0.0.1:50486, 127.0.0.1:50476]}
2022-09-26T15:00:13,818 - INFO - [Time-limited test:TestLedgerChecker@206] - unreplicated fragment: Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50466, 127.0.0.1:50474], Closed: true)
2022-09-26T15:00:13,819 - INFO - [Time-limited test:TestLedgerChecker@206] - unreplicated fragment: Fragment(LedgerID: 0, FirstEntryID: 1[-1], LastKnownEntryID: 1[-1], Host: [127.0.0.1:50466], Closed: false)
2022-09-26T15:00:13,819 - INFO - [Time-limited test:TestLedgerChecker@212] - bookies fragments [127.0.0.1:50466, 127.0.0.1:50474, 127.0.0.1:50476], [0, 1]
2022-09-26T15:00:13,819 - INFO - [Time-limited test:BookKeeperClusterTestCase@199] - TearDown
### When test execution failed:
2022-09-26T14:58:09,687 - INFO - [Time-limited test:LedgerChecker@373] - checkLedger nowing.
2022-09-26T14:58:09,689 - INFO - [Time-limited test:LedgerChecker@398] - LedgerCheck fragments before[Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50105, 127.0.0.1:50107, 127.0.0.1:50109], Closed: true)]
2022-09-26T14:58:09,690 - INFO - [Time-limited test:LedgerChecker@430] - not execute check fragments. curEnsemble [127.0.0.1:50105, 127.0.0.1:50122, 127.0.0.1:50107], curEntryId 1
**2022-09-26T14:58:09,704 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker$1@440] - check2Fragments [Fragment(LedgerID: 0, **FirstEntryID: 1[1]**, LastKnownEntryID: 1[1], Host: [127.0.0.1:50105, 127.0.0.1:50122, 127.0.0.1:50107], Closed: false), Fragment(LedgerID: 0, **FirstEntryID: 0[0]**, LastKnownEntryID: 0[0], Host: [127.0.0.1:50105, 127.0.0.1:50107, 127.0.0.1:50109], Closed: true)]**
2022-09-26T14:58:09,705 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker@479] - Checking fragment Fragment(LedgerID: 0, FirstEntryID: 1[1], LastKnownEntryID: 1[1], Host: [127.0.0.1:50105, 127.0.0.1:50122, 127.0.0.1:50107], Closed: false)
2022-09-26T14:58:09,707 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker@479] - Checking fragment Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50105, 127.0.0.1:50107, 127.0.0.1:50109], Closed: true)
2022-09-26T14:58:09,708 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker$FullLedgerCallback@360] - operationComplete Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50105, 127.0.0.1:50109], Closed: true), -8
2022-09-26T14:58:09,708 - INFO - [BookKeeperClientWorker-OrderedExecutor-0-0:LedgerChecker$FullLedgerCallback@360] - operationComplete Fragment(LedgerID: 0, FirstEntryID: 1[-1], LastKnownEntryID: 1[-1], Host: [127.0.0.1:50105], Closed: false), -8
**2022-09-26T14:58:09,709 - INFO - [Time-limited test:TestLedgerChecker@534] - waitAndGetResult [Fragment(LedgerID: 0, **FirstEntryID: 1[-1]**, LastKnownEntryID: 1[-1], Host: [127.0.0.1:50105], Closed: false), Fragment(LedgerID: 0, **FirstEntryID: 0[0]**, LastKnownEntryID: 0[0], Host: [127.0.0.1:50105, 127.0.0.1:50109], Closed: true)]**
2022-09-26T14:58:09,709 - INFO - [Time-limited test:TestLedgerChecker@201] - other Ensemble {0=[127.0.0.1:50105, 127.0.0.1:50109, 127.0.0.1:50107], 1=[127.0.0.1:50105, 127.0.0.1:50122, 127.0.0.1:50107]}
2022-09-26T14:58:09,709 - INFO - [Time-limited test:TestLedgerChecker@206] - unreplicated fragment: Fragment(LedgerID: 0, FirstEntryID: 1[-1], LastKnownEntryID: 1[-1], Host: [127.0.0.1:50105], Closed: false)
2022-09-26T14:58:09,709 - INFO - [Time-limited test:TestLedgerChecker@206] - unreplicated fragment: Fragment(LedgerID: 0, FirstEntryID: 0[0], LastKnownEntryID: 0[0], Host: [127.0.0.1:50105, 127.0.0.1:50109], Closed: true)
2022-09-26T14:58:09,709 - INFO - [Time-limited test:TestLedgerChecker@212] - bookies fragments [127.0.0.1:50105, 127.0.0.1:50122, 127.0.0.1:50107], [0]
2022-09-26T14:58:09,710 - INFO - [Time-limited test:BookKeeperClusterTestCase@199] - TearDown
### Changes
Replace `HashSet` with `LinkedHashSet`
(cherry picked from commit 74a87e260bd0cf9eca2d229cc181638741b5f575)
---
.../src/main/java/org/apache/bookkeeper/client/LedgerChecker.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
index 68c6adb0b0..b9df5f4a22 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
@@ -22,6 +22,7 @@ package org.apache.bookkeeper.client;
import io.netty.buffer.ByteBuf;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -344,7 +345,7 @@ public class LedgerChecker {
FullLedgerCallback(long numFragments,
GenericCallback<Set<LedgerFragment>> cb) {
- badFragments = new HashSet<LedgerFragment>();
+ badFragments = new LinkedHashSet<>();
this.numFragments = new AtomicLong(numFragments);
this.cb = cb;
}
@@ -376,7 +377,7 @@ public class LedgerChecker {
final GenericCallback<Set<LedgerFragment>> cb,
long percentageOfLedgerFragmentToBeVerified) {
// build a set of all fragment replicas
- final Set<LedgerFragment> fragments = new HashSet<LedgerFragment>();
+ final Set<LedgerFragment> fragments = new LinkedHashSet<>();
Long curEntryId = null;
List<BookieId> curEnsemble = null;
[bookkeeper] 28/33: Not wrap IOException twice form checkpoint (#3683)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit a8df522f0229c6826d23337e8f11ad22a169d35d
Author: wenbingshen <ol...@gmail.com>
AuthorDate: Wed Feb 15 15:15:11 2023 +0800
Not wrap IOException twice form checkpoint (#3683)
### Motivation
IOException may be wrapped by RuntimeException, so don't wrap RuntimeException with IOException.
(cherry picked from commit d6d9212a64a6c9dabd69a646082718929c43b4f9)
---
.../ldb/SingleDirectoryDbLedgerStorage.java | 12 ++---------
.../bookkeeper/bookie/storage/ldb/WriteCache.java | 5 +++--
.../bookie/storage/ldb/WriteCacheTest.java | 24 +++++++++++++++++++---
3 files changed, 26 insertions(+), 15 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
index fbc57823ba..d8646c1e43 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
@@ -731,12 +731,8 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
Batch batch = entryLocationIndex.newBatch();
writeCacheBeingFlushed.forEach((ledgerId, entryId, entry) -> {
- try {
- long location = entryLogger.addEntry(ledgerId, entry, true);
- entryLocationIndex.addLocation(batch, ledgerId, entryId, location);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ long location = entryLogger.addEntry(ledgerId, entry, true);
+ entryLocationIndex.addLocation(batch, ledgerId, entryId, location);
});
long entryLoggerStart = MathUtils.nowInNano();
@@ -789,10 +785,6 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
recordFailedEvent(dbLedgerStorageStats.getFlushStats(), startTime);
// Leave IOExecption as it is
throw e;
- } catch (RuntimeException e) {
- recordFailedEvent(dbLedgerStorageStats.getFlushStats(), startTime);
- // Wrap unchecked exceptions
- throw new IOException(e);
} finally {
try {
isFlushOngoing.set(false);
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java
index 659162e46d..b3f37671ce 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCache.java
@@ -27,6 +27,7 @@ import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.Unpooled;
import java.io.Closeable;
+import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReentrantLock;
@@ -57,7 +58,7 @@ public class WriteCache implements Closeable {
* Consumer that is used to scan the entire write cache.
*/
public interface EntryConsumer {
- void accept(long ledgerId, long entryId, ByteBuf entry);
+ void accept(long ledgerId, long entryId, ByteBuf entry) throws IOException;
}
private final ConcurrentLongLongPairHashMap index = ConcurrentLongLongPairHashMap.newBuilder()
@@ -220,7 +221,7 @@ public class WriteCache implements Closeable {
private static final ArrayGroupSort groupSorter = new ArrayGroupSort(2, 4);
- public void forEach(EntryConsumer consumer) {
+ public void forEach(EntryConsumer consumer) throws IOException {
sortedEntriesLock.lock();
try {
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java
index e92f0e755e..cd433e8fd2 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/WriteCacheTest.java
@@ -30,7 +30,8 @@ import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.Unpooled;
import io.netty.buffer.UnpooledByteBufAllocator;
-
+import io.netty.util.ReferenceCountUtil;
+import java.io.IOException;
import java.nio.charset.Charset;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CountDownLatch;
@@ -144,7 +145,7 @@ public class WriteCacheTest {
}
@Test
- public void testEmptyCache() {
+ public void testEmptyCache() throws IOException {
WriteCache cache = new WriteCache(allocator, 1024 * 1024, 16 * 1024);
assertEquals(0, cache.count());
@@ -222,7 +223,7 @@ public class WriteCacheTest {
}
@Test
- public void testLedgerDeletion() {
+ public void testLedgerDeletion() throws IOException {
WriteCache cache = new WriteCache(allocator, 1024 * 1024, 16 * 1024);
ByteBuf entry = Unpooled.buffer(1024);
@@ -306,4 +307,21 @@ public class WriteCacheTest {
}
assertFalse(cache.hasEntry(ledgerId, 48));
}
+
+ @Test(expected = IOException.class)
+ public void testForEachIOException() throws Exception {
+ try (WriteCache cache = new WriteCache(allocator, 1024 * 1024, 16 * 1024)) {
+
+ for (int i = 0; i < 48; i++) {
+ boolean inserted = cache.put(1, i, Unpooled.wrappedBuffer(("test-" + i).getBytes()));
+ assertTrue(inserted);
+ }
+
+ assertEquals(48, cache.count());
+
+ cache.forEach(((ledgerId, entryId, entry) -> {
+ throw new IOException("test throw IOException.");
+ }));
+ }
+ }
}
[bookkeeper] 01/33: Revert "InterleavedLedgerStorage: safe release ByteBuf (#3674)"
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit c74c2592de93f9719dde53da3433226bcf83c2f4
Author: zymap <zh...@gmail.com>
AuthorDate: Thu Feb 16 17:21:46 2023 +0800
Revert "InterleavedLedgerStorage: safe release ByteBuf (#3674)"
This reverts commit 1faaaf2ad4447859677205f49fbf5321e879f774.
---
.../java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
index ff79b7234f..a1dc14b2f6 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
@@ -37,7 +37,7 @@ import com.google.common.util.concurrent.RateLimiter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
-import io.netty.util.ReferenceCountUtil;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -348,7 +348,7 @@ public class InterleavedLedgerStorage implements CompactableLedgerStorage, Entry
lac = bb.readLong();
lac = ledgerCache.updateLastAddConfirmed(ledgerId, lac);
} finally {
- ReferenceCountUtil.safeRelease(bb);
+ bb.release();
}
}
}
[bookkeeper] 04/33: Adjust log stack output (#3496)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 63a90c75bd927b7a52762088e705bc616b90c146
Author: wenbingshen <ol...@gmail.com>
AuthorDate: Mon Oct 17 12:37:29 2022 +0800
Adjust log stack output (#3496)
### Motivation
The default `dnsResolver` of `RackawareEnsemblePlacementPolicyImpl` is initialized as `ScriptBasedMapping`. Since the script is missing by default, detailed stack information will be printed, but the code will fall back to `DefaultResolver` by default, so only simple exception information needs to be printed.
(cherry picked from commit cbb74eab4f1f464c0e4760acf9304af2289acfa1)
---
.../bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
index 5185dc4753..c841044a03 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
@@ -269,8 +269,8 @@ public class RackawareEnsemblePlacementPolicyImpl extends TopologyAwareEnsembleP
}
} catch (RuntimeException re) {
if (!conf.getEnforceMinNumRacksPerWriteQuorum()) {
- LOG.warn("Failed to initialize DNS Resolver {}, used default subnet resolver ",
- dnsResolverName, re);
+ LOG.warn("Failed to initialize DNS Resolver {}, used default subnet resolver because {}",
+ dnsResolverName, re.getMessage());
dnsResolver = new DefaultResolver(this::getDefaultRack);
dnsResolver.setBookieAddressResolver(bookieAddressResolver);
} else {
[bookkeeper] 14/33: BookieImpl remove wait until journal quits (#3603)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 7614cd6bfdc2ea5a0a177c8087d61af6517593d8
Author: wenbingshen <ol...@gmail.com>
AuthorDate: Thu Feb 9 15:30:52 2023 +0800
BookieImpl remove wait until journal quits (#3603)
### Motivation
PR #2887 introduced the feature of shutdown the bookie service after any Journal thread exits, so we don't need to wait in BookieImpl for all Journal threads to exit before shutdown the Bookie service, because this cannot happen.
(cherry picked from commit b85ac48e8b61748300206646ab9dd1fc16ed1616)
---
.../org/apache/bookkeeper/bookie/BookieImpl.java | 27 +++-------------------
.../apache/bookkeeper/test/ReadOnlyBookieTest.java | 2 +-
2 files changed, 4 insertions(+), 25 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 03cd6ad681..1d000ab53a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -781,30 +781,9 @@ public class BookieImpl extends BookieCriticalThread implements Bookie {
@Override
public void run() {
- // bookie thread wait for journal thread
- try {
- // start journals
- for (Journal journal: journals) {
- journal.start();
- }
-
- // wait until journal quits
- for (Journal journal: journals) {
-
- journal.joinThread();
- }
- LOG.info("Journal thread(s) quit.");
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted on running journal thread : ", ie);
- }
- // if the journal thread quits due to shutting down, it is ok
- if (!stateManager.isShuttingDown()) {
- // some error found in journal thread and it quits
- // following add operations to it would hang unit client timeout
- // so we should let bookie server exists
- LOG.error("Journal manager quits unexpectedly.");
- triggerBookieShutdown(ExitCode.BOOKIE_EXCEPTION);
+ // start journals
+ for (Journal journal: journals) {
+ journal.start();
}
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java
index cf03cf6629..e8cdbd590d 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java
@@ -231,7 +231,7 @@ public class ReadOnlyBookieTest extends BookKeeperClusterTestCase {
assertEquals("writable dirs should have one dir", 1, ledgerDirsManager
.getWritableLedgerDirs().size());
assertTrue("Bookie should shutdown if readOnlyMode not enabled",
- bookie.isAlive());
+ bookie.isRunning());
}
private void startNewBookieWithMultipleLedgerDirs(int numOfLedgerDirs)
[bookkeeper] 02/33: Revert "[refactor][bookkeeper] Refactor ByteBuf release method in DefaultEntryLogger (#3673)"
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 4543493a000b9c25ae41d5d867a9470a3b0e8e95
Author: zymap <zh...@gmail.com>
AuthorDate: Thu Feb 16 17:21:56 2023 +0800
Revert "[refactor][bookkeeper] Refactor ByteBuf release method in DefaultEntryLogger (#3673)"
This reverts commit 6b37505e35898a4fee35ad320ea44d02374203bd.
---
.../main/java/org/apache/bookkeeper/bookie/EntryLogger.java | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index 396410d14e..d69e434ef5 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -32,7 +32,6 @@ import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.Unpooled;
-import io.netty.util.ReferenceCountUtil;
import io.netty.util.concurrent.FastThreadLocal;
import java.io.BufferedReader;
import java.io.File;
@@ -189,7 +188,7 @@ public class EntryLogger {
throw e;
}
} finally {
- ReferenceCountUtil.safeRelease(serializedMap);
+ serializedMap.release();
}
// Flush the ledger's map out before we write the header.
// Otherwise the header might point to something that is not fully
@@ -858,7 +857,7 @@ public class EntryLogger {
ByteBuf data = allocator.buffer(entrySize, entrySize);
int rc = readFromLogChannel(entryLogId, fc, data, pos);
if (rc != entrySize) {
- ReferenceCountUtil.safeRelease(data);
+ data.release();
throw new IOException("Bad entry read from log file id: " + entryLogId,
new EntryLookupException("Short read for " + ledgerId + "@"
+ entryId + " in " + entryLogId + "@"
@@ -896,7 +895,7 @@ public class EntryLogger {
int ledgersCount = headers.readInt();
return new Header(headerVersion, ledgersMapOffset, ledgersCount);
} finally {
- ReferenceCountUtil.safeRelease(headers);
+ headers.release();
}
}
@@ -1042,7 +1041,7 @@ public class EntryLogger {
pos += entrySize;
}
} finally {
- ReferenceCountUtil.safeRelease(data);
+ data.release();
}
}
@@ -1145,7 +1144,7 @@ public class EntryLogger {
} catch (IndexOutOfBoundsException e) {
throw new IOException(e);
} finally {
- ReferenceCountUtil.safeRelease(ledgersMap);
+ ledgersMap.release();
}
if (meta.getLedgersMap().size() != header.ledgersCount) {
[bookkeeper] 17/33: [log] Add journal file path that caused failure in multi-journal config (#3623)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit c0ac37748466b4d3ae7d4d572ed9ed484364299c
Author: Rajan Dhabalia <rd...@apache.org>
AuthorDate: Tue Nov 8 17:08:59 2022 -0800
[log] Add journal file path that caused failure in multi-journal config (#3623)
### Motivation
Right now, if bookie fails to replay the journal then it logs journal-txn-id but it doesn't log journal dir path. It will be useful if bookie prints journal path along with journal-dir in case if user is using multi-journal directories so, user can troubleshoot the issue and fix it correctly.
```
2022-11-07T18:29:07,205+0000 [main] ERROR org.apache.bookkeeper.bookie.Bookie - Exception while replaying journals, shutting down
java.io.IOException: Recovery log 1667603421612 is missing
at org.apache.bookkeeper.bookie.Bookie.replay(Bookie.java:932) ~[org.apache.bookkeeper-bookkeeper-server-4.12.jar:4.12]
at org.apache.bookkeeper.bookie.Bookie.readJournal(Bookie.java:911) ~[org.apache.bookkeeper-bookkeeper-server-4.12.jar]
```
### Changes
Add journal dir file path in journal replay failure.
(cherry picked from commit 8357e4b00f0b2244dbe7648302dd02d930e5ecb7)
---
.../src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 1d000ab53a..6bc0818577 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -609,7 +609,8 @@ public class BookieImpl extends BookieCriticalThread implements Bookie {
// validate filtered log ids only when we have markedLogId
if (markedLog.getLogFileId() > 0) {
if (logs.size() == 0 || logs.get(0) != markedLog.getLogFileId()) {
- throw new IOException("Recovery log " + markedLog.getLogFileId() + " is missing");
+ String path = journal.getJournalDirectory().getAbsolutePath();
+ throw new IOException("Recovery log " + markedLog.getLogFileId() + " is missing at " + path);
}
}
[bookkeeper] 31/33: recycle LongPairWrapper (#3728)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit f92171f10962531103094a84b65da2e57b137475
Author: wenbingshen <ol...@gmail.com>
AuthorDate: Fri Feb 10 17:26:34 2023 +0800
recycle LongPairWrapper (#3728)
(cherry picked from commit cec8267682202dfb77342db4366fef07ad7632d6)
---
.../bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
index e15457ca8e..a3d4df0181 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
@@ -105,7 +105,13 @@ public class LocationsIndexRebuildOp {
// Update the ledger index page
LongPairWrapper key = LongPairWrapper.get(ledgerId, entryId);
LongWrapper value = LongWrapper.get(location);
- batch.get().put(key.array, value.array);
+
+ try {
+ batch.get().put(key.array, value.array);
+ } finally {
+ key.recycle();
+ value.recycle();
+ }
if (count.incrementAndGet() > BATCH_COMMIT_SIZE) {
batch.get().flush();
[bookkeeper] 11/33: [configuration] fix typo in bk_server.conf (#3574)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit fbe0e5f08fabb84d49e1f7883b36151d751869ec
Author: FeiFan Wang <zo...@163.com>
AuthorDate: Wed Oct 26 22:43:12 2022 +0800
[configuration] fix typo in bk_server.conf (#3574)
(cherry picked from commit da4583e9390352d3414cc1b954591926034431bb)
---
conf/bk_server.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index 68c4ee2e29..f72f54a676 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -799,7 +799,7 @@ metadataServiceUri=zk+hierarchical://localhost:2181/ledgers
# store all ledgers.
# zkLedgersRootPath=/ledgers
-# @Deprecated - `zkLedgersRootPath` is deprecated in favor of using `metadataServiceUri`
+# @Deprecated - `zkServers` is deprecated in favor of using `metadataServiceUri`
# A list of one of more servers on which Zookeeper is running.
# The server list can be comma separated values, for example:
# zkServers=zk1:2181,zk2:2181,zk3:2181
[bookkeeper] 32/33: Fix memory leak when the Bookie is in read-only mode. (#3746)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit fcff85bebf0290b4d3ae79798f9ea52e994dbf18
Author: Yan Zhao <ho...@apache.org>
AuthorDate: Wed Feb 8 14:53:11 2023 +0800
Fix memory leak when the Bookie is in read-only mode. (#3746)
Descriptions of the changes in this PR:
Fixes #3745
(cherry picked from commit af82d14cb1da6a97b563b80e80b5d2fe462e2c4f)
---
.../main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java | 2 +-
.../main/java/org/apache/bookkeeper/proto/PacketProcessorBase.java | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
index 860fb6635f..88ee09d61f 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
@@ -180,7 +180,7 @@ public class BookieProtoEncoding {
packet.markReaderIndex();
return BookieProtocol.ParsedAddRequest.create(
version, ledgerId, entryId, flags,
- masterKey, packet.retain());
+ masterKey, packet);
}
case BookieProtocol.READENTRY:
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PacketProcessorBase.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PacketProcessorBase.java
index 0fb2d3f8f0..5bee27b0fb 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PacketProcessorBase.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PacketProcessorBase.java
@@ -176,7 +176,9 @@ abstract class PacketProcessorBase<T extends Request> extends SafeRunnable {
if (request instanceof BookieProtocol.ReadRequest) {
requestProcessor.onReadRequestFinish();
}
- if (request instanceof BookieProtocol.AddRequest) {
+ if (request instanceof BookieProtocol.ParsedAddRequest) {
+ ((BookieProtocol.ParsedAddRequest) request).release();
+ request.recycle();
requestProcessor.onAddRequestFinish();
}
return;
[bookkeeper] 03/33: Bump jcommander from 1.78 to 1.82 (#3476)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 5fdef3321740bef0d9712f3f68b790bffb235c00
Author: ZhangJian He <sh...@gmail.com>
AuthorDate: Wed Sep 7 22:38:18 2022 +0800
Bump jcommander from 1.78 to 1.82 (#3476)
### Motivation
That's a regular update, `jcommander` 1.78 was released in 2019.
### Modifications
Bump jcommander version from 1.78 to 1.82
(cherry picked from commit eadac6297ec14455f3c7a3a02ceb48382cebbd25)
---
bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 4 ++--
bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt | 4 ++--
bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 4 ++--
bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt | 2 +-
bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt | 2 +-
bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt | 2 +-
pom.xml | 2 +-
7 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index b4096508d1..52501f29c2 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -265,7 +265,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-util-9.4.48.v20220622.jar [22]
- lib/org.eclipse.jetty-jetty-util-ajax-9.4.48.v20220622.jar [22]
- lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23]
-- lib/com.beust-jcommander-1.78.jar [24]
+- lib/com.beust-jcommander-1.82.jar [24]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
- lib/net.jpountz.lz4-lz4-1.3.0.jar [26]
@@ -337,7 +337,7 @@ Apache Software License, Version 2.
[21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
[22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
[23] Source available at https://github.com/facebook/rocksdb/tree/v6.22.1
-[24] Source available at https://github.com/cbeust/jcommander/tree/1.78
+[24] Source available at https://github.com/cbeust/jcommander/tree/1.82
[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
[28] Source available at https://github.com/googleapis/googleapis
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
index 2e65c9a4c0..4e10bed628 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
@@ -246,7 +246,7 @@ Apache Software License, Version 2.
- lib/org.apache.zookeeper-zookeeper-3.8.0.jar [20]
- lib/org.apache.zookeeper-zookeeper-jute-3.8.0.jar [20]
- lib/org.apache.zookeeper-zookeeper-3.8.0-tests.jar [20]
-- lib/com.beust-jcommander-1.78.jar [23]
+- lib/com.beust-jcommander-1.82.jar [23]
- lib/net.jpountz.lz4-lz4-1.3.0.jar [25]
- lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [27]
- lib/com.google.code.gson-gson-2.8.9.jar [28]
@@ -306,7 +306,7 @@ Apache Software License, Version 2.
[18] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-collections.git;a=tag;h=a3a5ad
[19] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=shortlog;h=refs/tags/LANG_3_6
[20] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
-[23] Source available at https://github.com/cbeust/jcommander/tree/1.78
+[23] Source available at https://github.com/cbeust/jcommander/tree/1.82
[25] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
[27] Source available at https://github.com/googleapis/googleapis
[28] Source available at https://github.com/google/gson/tree/gson-parent-2.8.9
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
index eac2df92e7..e6eacd3a46 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
@@ -265,7 +265,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-util-9.4.48.v20220622.jar [22]
- lib/org.eclipse.jetty-jetty-util-ajax-9.4.48.v20220622.jar [22]
- lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23]
-- lib/com.beust-jcommander-1.78.jar [24]
+- lib/com.beust-jcommander-1.82.jar [24]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
- lib/net.jpountz.lz4-lz4-1.3.0.jar [26]
@@ -334,7 +334,7 @@ Apache Software License, Version 2.
[21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
[22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
[23] Source available at https://github.com/facebook/rocksdb/tree/v6.16.4
-[24] Source available at https://github.com/cbeust/jcommander/tree/1.78
+[24] Source available at https://github.com/cbeust/jcommander/tree/1.82
[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
[28] Source available at https://github.com/googleapis/googleapis
diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt
index 0dd3a7f2dc..91f4e5dde2 100644
--- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt
@@ -123,7 +123,7 @@ Permission to use, copy, modify and distribute UnixCrypt
for non-commercial or commercial purposes and without fee is
granted provided that the copyright notice appears in all copies.
------------------------------------------------------------------------------------
-- lib/com.beust-jcommander-1.78.jar
+- lib/com.beust-jcommander-1.82.jar
Copyright 2010 Cedric Beust cedric@beust.com
diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt
index ba9d06031c..a7e09fb5fe 100644
--- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt
+++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt
@@ -49,7 +49,7 @@ License for the specific language governing permissions and limitations
under the License.
------------------------------------------------------------------------------------
-- lib/com.beust-jcommander-1.78.jar
+- lib/com.beust-jcommander-1.82.jar
Copyright 2010 Cedric Beust cedric@beust.com
diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt
index 9466ad92d1..fe0761694d 100644
--- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt
@@ -105,7 +105,7 @@ Permission to use, copy, modify and distribute UnixCrypt
for non-commercial or commercial purposes and without fee is
granted provided that the copyright notice appears in all copies.
------------------------------------------------------------------------------------
-- lib/com.beust-jcommander-1.78.jar
+- lib/com.beust-jcommander-1.82.jar
Copyright 2010 Cedric Beust cedric@beust.com
diff --git a/pom.xml b/pom.xml
index e97a9d0232..608e6bb9e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -142,7 +142,7 @@
<hamcrest.version>1.3</hamcrest.version>
<hdrhistogram.version>2.1.10</hdrhistogram.version>
<jackson.version>2.13.4.20221013</jackson.version>
- <jcommander.version>1.78</jcommander.version>
+ <jcommander.version>1.82</jcommander.version>
<jetty.version>9.4.48.v20220622</jetty.version>
<jmh.version>1.19</jmh.version>
<jmock.version>2.8.2</jmock.version>
[bookkeeper] 19/33: Upgrade docker image version to fix CVEs (#3640)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit fba90caa4911f0610fc78882066ae7bf9494189f
Author: Hang Chen <ch...@apache.org>
AuthorDate: Tue Nov 15 16:29:14 2022 +0800
Upgrade docker image version to fix CVEs (#3640)
### Motivation
There are a lot of critical CVEs in the `maven:3.8.6-jdk-11` docker image
https://snyk.io/test/docker/maven%3A3.8.6-jdk-11
### Changes
Use `maven:3.8.3-eclipse-temurin-11` instead of `maven:3.8.6-jdk-11` to fix it.
(cherry picked from commit fbad323775f72ae2e8154892b489801305860bb1)
---
dev/docker/Dockerfile | 2 +-
dev/release/Dockerfile | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/dev/docker/Dockerfile b/dev/docker/Dockerfile
index 09a1186fd1..63102a939c 100644
--- a/dev/docker/Dockerfile
+++ b/dev/docker/Dockerfile
@@ -17,7 +17,7 @@
# under the License.
#
-FROM maven:3.8.6-jdk-11
+FROM maven:3.8.6-eclipse-temurin-11
RUN apt-get update
RUN apt-get install -y g++ cmake
diff --git a/dev/release/Dockerfile b/dev/release/Dockerfile
index 2bcd90a40b..0c349d9b5f 100644
--- a/dev/release/Dockerfile
+++ b/dev/release/Dockerfile
@@ -17,7 +17,7 @@
# under the License.
#
-FROM maven:3.8.6-jdk-8
+FROM maven:3.8.6-eclipse-temurin-8
RUN apt-get update
RUN apt-get install -y g++ cmake gnupg2 vim subversion
[bookkeeper] 21/33: Check client if closed when complete callback. (#3661)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit 773105e187e3a7ff25ddcabecf11830bb61195bd
Author: Qiang Zhao <ma...@apache.org>
AuthorDate: Thu Nov 24 10:53:12 2022 +0800
Check client if closed when complete callback. (#3661)
### Motivation
Follow the same rules to complete the callback to avoid Bookkeeper client operations being allowed even after its closure.
### Changes
- Use `bk.getReturnRc(rc)` check the client state.
(cherry picked from commit 221cba4ed32980e6e3109e9b9cca7e380b64ee65)
---
.../src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
index 847da0d89c..5cb15da2c7 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerOpenOp.java
@@ -208,7 +208,7 @@ class LedgerOpenOp {
}
if (rc == BKException.Code.UnauthorizedAccessException
|| rc == BKException.Code.TimeoutException) {
- openComplete(rc, null);
+ openComplete(bk.getReturnRc(rc), null);
} else {
openComplete(bk.getReturnRc(BKException.Code.LedgerRecoveryException), null);
}
[bookkeeper] 13/33: update doc about flushInterval config (#3601)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit e32e327a54c04c5314a88a5975faea367e732949
Author: LinChen <15...@qq.com>
AuthorDate: Fri Nov 4 15:00:41 2022 +0800
update doc about flushInterval config (#3601)
* update flushInterval doc
(cherry picked from commit 860d40d58ca0eea1eaca2ddb81b43cb80447ae73)
---
conf/bk_server.conf | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index f72f54a676..a241a31a2b 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -463,7 +463,8 @@ ledgerDirectories=/tmp/bk-data
# on too frequent flushing. You can consider increment flush interval
# to get better performance, but you need to pay more time on bookie
# server restart after failure.
-# This config is used only when entryLogPerLedgerEnabled is enabled.
+# This config is used when entryLogPerLedgerEnabled=true
+# or ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.
# flushInterval=10000
# Allow the expansion of bookie storage capacity. Newly added ledger