You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bifromq.apache.org by po...@apache.org on 2025/11/12 01:18:03 UTC
(bifromq) branch main updated: Update merge protocol & various improvements for production stability (#188)
This is an automated email from the ASF dual-hosted git repository.
popduke pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/bifromq.git
The following commit(s) were added to refs/heads/main by this push:
new 2b342e989 Update merge protocol & various improvements for production stability (#188)
2b342e989 is described below
commit 2b342e989ab8031e66f2530d297f45df6c81d6a3
Author: Yonny(Yu) Hao <po...@gmail.com>
AuthorDate: Wed Nov 12 09:17:58 2025 +0800
Update merge protocol & various improvements for production stability (#188)
Base-KV improvements:
Loosens the merge prerequisite on config alignment, enhances the RedundantRangeRemovalBalancer to detect and purge zombie ranges, refactors the local engine for pluggability, and rolls out multiple optimizations—snapshot session reuse, obsolete snapshot filtering, leaner installation flow, lower split/merge memory pressure, faster distributed range lookup, RocksDB internal metrics toggles, redefined WAL compaction thresholds, improved WAL read path, and better default local-engine set [...]
Inbox service optimizations:
Cuts read/write overhead in InboxStoreCoProc and accelerates BatchInsertRequest marshalling.
Multi-tenancy enhancements:
Adds tenant-level observability metrics and introduces a tenant-level switch to publish will messages during shutdown.
Bug fixes:
Resolves a NoSuchElementException when parsing AgentMetadata from CRDT and eliminates task duplication in BatchQueryCall.
---
.github/workflows/build-cov.yaml | 33 +-
.github/workflows/build-dev.yaml | 33 +-
.licenserc.yaml | 3 +
.../basecluster/memberlist/HostMemberList.java | 5 +
.../basecluster/memberlist/agent/CRDTUtil.java | 17 +-
.../bifromq/basecluster/messenger/Messenger.java | 55 +-
.../basecluster/messenger/MessengerTransport.java | 5 +-
.../basecluster/transport/UDPTransport.java | 12 +-
.../basecluster/memberlist/HostMemberListTest.java | 61 +
.../bifromq/basecrdt/service/CRDTCluster.java | 8 +-
.../org/apache/bifromq/baseenv/ZeroCopyParser.java | 16 +-
.../pom.xml | 56 +-
.../memory/AbstractInMemKVSpaceReader.java} | 38 +-
.../localengine/memory/InMemCPableKVEngine.java | 16 +-
.../localengine/memory/InMemCPableKVSpace.java | 194 +++
.../localengine/memory/InMemDefaultConfigs.java} | 16 +-
.../basekv/localengine/memory/InMemKVEngine.java | 9 +-
.../localengine/memory/InMemKVEngineProvider.java} | 34 +-
.../basekv/localengine/memory/InMemKVHelper.java | 46 +
.../basekv/localengine/memory/InMemKVSpace.java | 145 ++
.../localengine/memory/InMemKVSpaceCheckpoint.java | 33 +-
.../memory/InMemKVSpaceCheckpointReader.java | 67 +
.../localengine/memory/InMemKVSpaceEpoch.java | 88 ++
.../localengine/memory/InMemKVSpaceIterator.java | 71 +-
.../memory/InMemKVSpaceMigratableWriter.java | 67 +
.../localengine/memory/InMemKVSpaceReader.java | 121 ++
.../localengine/memory/InMemKVSpaceWriter.java | 98 +-
.../memory/InMemKVSpaceWriterHelper.java | 116 +-
.../localengine/memory/InMemWALableKVEngine.java | 17 +-
.../localengine/memory/InMemWALableKVSpace.java | 50 +-
...ifromq.basekv.localengine.spi.IKVEngineProvider | 1 +
...InMemCPableKVSpaceRestoreFlushListenerTest.java | 78 +
.../localengine/memory/InMemKVEngineTest.java | 11 +-
.../src/test/resources/log4j2-test.xml | 1 +
.../pom.xml | 52 +-
.../rocksdb/AbstractRocksDBKVSpaceReader.java | 66 +
.../localengine/rocksdb}/AdaptiveWriteBudget.java | 4 +-
.../basekv/localengine/rocksdb/AutoCleaner.java | 0
.../rocksdb/IRocksDBKVSpaceCheckpoint.java | 0
.../localengine/rocksdb/IRocksDBKVSpaceEpoch.java} | 35 +-
.../rocksdb/IRocksDBKVSpaceEpochHandle.java} | 13 +-
.../localengine/rocksdb/IWriteStatsRecorder.java | 0
.../localengine/rocksdb/IteratorOptions.java} | 6 +-
.../bifromq/basekv/localengine/rocksdb/Keys.java | 0
.../rocksdb/NoopWriteStatsRecorder.java | 0
.../localengine/rocksdb/RocksDBCPableKVEngine.java | 39 +-
.../localengine/rocksdb/RocksDBCPableKVSpace.java | 556 +++++++
.../rocksdb/RocksDBCPableKVSpaceEpochHandle.java | 68 +
.../localengine/rocksdb/RocksDBDefaultConfigs.java | 93 ++
.../basekv/localengine/rocksdb/RocksDBHelper.java | 126 ++
.../localengine/rocksdb/RocksDBKVEngine.java | 30 +-
.../rocksdb/RocksDBKVEngineIterator.java | 18 +-
.../rocksdb/RocksDBKVEngineProvider.java} | 41 +-
.../basekv/localengine/rocksdb/RocksDBKVSpace.java | 214 +++
.../rocksdb/RocksDBKVSpaceCheckpoint.java | 89 +-
.../rocksdb/RocksDBKVSpaceCheckpointReader.java | 74 +
.../rocksdb/RocksDBKVSpaceCompactionTrigger.java | 4 +-
.../rocksdb/RocksDBKVSpaceEpochHandle.java | 401 +++++
.../rocksdb/RocksDBKVSpaceIterator.java | 205 +++
.../rocksdb/RocksDBKVSpaceMigratableWriter.java | 71 +
.../localengine/rocksdb/RocksDBKVSpaceReader.java | 102 ++
.../localengine/rocksdb/RocksDBKVSpaceWriter.java | 163 ++
.../rocksdb/RocksDBKVSpaceWriterHelper.java | 34 +-
.../localengine/rocksdb/RocksDBOptionsUtil.java} | 254 ++--
.../localengine/rocksdb/RocksDBSnapshot.java} | 21 +-
.../rocksdb/RocksDBWALableKVEngine.java | 30 +-
.../localengine/rocksdb/RocksDBWALableKVSpace.java | 92 +-
.../rocksdb/RocksDBWALableKVSpaceEpochHandle.java | 58 +
.../rocksdb/metrics/RocksDBKVSpaceMetric.java | 99 ++
...ifromq.basekv.localengine.spi.IKVEngineProvider | 1 +
.../localengine/benchmark/BenchmarkState.java | 24 +-
.../localengine/benchmark/BenchmarkTemplate.java | 22 +-
.../ContinuousKeyDeleteAndGet_Benchmark.java | 37 +-
.../ContinuousKeyDeleteAndSeek_Benchmark.java | 55 +-
.../ContinuousKeySingleDeleteAndSeek.java | 4 +-
.../ContinuousKeySingleDeleteAndSeekState.java | 13 +-
.../localengine/benchmark/HybridWorkload.java | 0
.../localengine/benchmark/HybridWorkloadState.java | 26 +-
.../benchmark/SingleKeyUpdateAndGet.java | 5 +-
.../benchmark/SingleKeyUpdateAndGetState.java | 5 +-
.../localengine/metrics/KVSpaceMetersTest.java | 149 ++
.../rocksdb/AbstractRawRocksDBTest.java | 10 +-
.../rocksdb/AbstractRocksDBCPableEngineTest.java} | 48 +-
.../rocksdb/AbstractRocksDBWALableEngineTest.java} | 71 +-
.../rocksdb/AdaptiveWriteBudgetTest.java | 86 ++
.../localengine/rocksdb/KVSpaceCheckpointTest.java | 4 +-
.../localengine/rocksdb/OverrideIdentityTest.java | 59 +-
.../rocksdb/RocksDBCPableKVEngineTest.java | 22 +-
.../RocksDBCPableKVSpaceCleanupInactiveTest.java | 99 ++
.../RocksDBCPableKVSpaceFreshBootstrapTest.java | 97 ++
.../RocksDBCPableKVSpaceLegacyMigrationTest.java | 124 ++
.../RocksDBCPableKVSpacePointerRecoveryTest.java | 138 ++
.../RocksDBCPableKVSpaceRestoreAbortTest.java | 98 ++
.../RocksDBCPableKVSpaceRestoreBulkTest.java | 101 ++
...cksDBCPableKVSpaceRestoreFlushListenerTest.java | 137 ++
.../RocksDBCPableKVSpaceRestoreOverlayTest.java | 117 ++
.../RocksDBCPableKVSpaceRestoreReplaceTest.java | 128 ++
...ksDBCPableKVSpaceRestoreRestartCleanupTest.java | 111 ++
...DBCPableKVSpaceRestoreSessionLifecycleTest.java | 100 ++
...cksDBCPableKVSpaceRestoreSwitchPointerTest.java | 100 ++
.../RocksDBCPableKVSpaceRuntimeCleanerTest.java | 125 ++
.../RocksDBKVSpaceCompactionTriggerTest.java | 2 +-
.../rocksdb/RocksDBWALableKVEngineTest.java | 18 +-
.../RocksDBWALableKVSpaceNoPointerTest.java | 81 +
.../src/test/resources/log4j2-test.xml | 1 +
.../pom.xml | 30 +-
.../basekv/localengine/AbstractKVEngine.java | 62 +-
.../basekv/localengine/AbstractKVSpace.java | 151 ++
.../basekv/localengine/AbstractKVSpaceReader.java | 50 +-
.../bifromq/basekv/localengine/ICPableKVSpace.java | 23 +-
.../bifromq/basekv/localengine/IKVEngine.java | 0
.../bifromq/basekv/localengine/IKVSpace.java | 29 +-
.../basekv/localengine/IKVSpaceCheckpoint.java} | 18 +-
.../basekv/localengine/IKVSpaceDataUpdatable.java | 17 +-
.../bifromq/basekv/localengine/IKVSpaceEpoch.java} | 5 +-
.../basekv/localengine/IKVSpaceIdentifiable.java} | 11 +-
.../basekv/localengine/IKVSpaceIterator.java | 45 +-
.../localengine/IKVSpaceMetadataUpdatable.java | 2 +-
.../basekv/localengine/IKVSpaceMetadataWriter.java | 0
.../basekv/localengine/IKVSpaceMigratable.java} | 22 +-
.../localengine/IKVSpaceMigratableWriter.java} | 3 +-
.../basekv/localengine/IKVSpaceReader.java} | 52 +-
.../localengine/IKVSpaceRefreshableReader.java} | 13 +-
.../basekv/localengine/IKVSpaceSizeable.java} | 11 +-
.../basekv/localengine/IKVSpaceWriteable.java | 0
.../bifromq/basekv/localengine/IKVSpaceWriter.java | 2 +-
.../basekv/localengine/IRestoreSession.java} | 31 +-
.../bifromq/basekv/localengine/ISyncContext.java | 83 +
.../basekv/localengine/IWALableKVSpace.java | 9 +-
.../basekv/localengine/KVEngineException.java | 0
.../basekv/localengine/KVEngineFactory.java | 49 +
.../basekv/localengine/KVSpaceDescriptor.java | 0
.../bifromq/basekv/localengine/RestoreMode.java} | 7 +-
.../bifromq/basekv/localengine/StructUtil.java | 90 ++
.../bifromq/basekv/localengine/SyncContext.java | 46 +-
.../localengine/metrics/GeneralKVSpaceMetric.java | 7 +-
.../basekv/localengine/metrics/IKVSpaceMetric.java | 2 +
.../basekv/localengine/metrics/KVSpaceMeters.java | 98 +-
.../localengine/metrics/KVSpaceOpMeters.java | 0
.../basekv/localengine/spi/IKVEngineProvider.java | 68 +
.../localengine/AbstractCPableEngineTest.java | 74 +
.../basekv/localengine/AbstractKVEngineTest.java | 300 ++++
.../localengine/AbstractWALableEngineTest.java} | 13 +-
.../bifromq/basekv/localengine/MockableTest.java | 0
.../basekv/localengine/SyncContextTest.java | 24 +-
.../bifromq/basekv/localengine/TestUtil.java | 0
.../localengine/metrics/KVSpaceMetersTest.java | 152 ++
.../src/test/resources/log4j2-test.xml | 0
.../basekv/localengine/IKVSpaceCheckpoint.java | 24 -
.../bifromq/basekv/localengine/ISyncContext.java | 38 -
.../basekv/localengine/KVEngineFactory.java | 52 -
.../localengine/memory/InMemCPableKVSpace.java | 79 -
.../memory/InMemKVEngineConfigurator.java | 42 -
.../basekv/localengine/memory/InMemKVSpace.java | 131 --
.../localengine/memory/InMemKVSpaceReader.java | 86 --
.../localengine/rocksdb/RocksDBCPableKVSpace.java | 236 ---
.../basekv/localengine/rocksdb/RocksDBKVSpace.java | 387 -----
.../rocksdb/RocksDBKVSpaceIterator.java | 136 --
.../localengine/rocksdb/RocksDBKVSpaceReader.java | 98 --
.../rocksdb/RocksDBKVSpaceSnapshot.java | 138 --
.../localengine/rocksdb/RocksDBKVSpaceWriter.java | 236 ---
.../RocksDBWALableKVEngineConfigurator.java | 48 -
.../rocksdb/metrics/RocksDBKVSpaceMetric.java | 58 -
.../basekv/localengine/AbstractKVEngineTest.java | 377 -----
.../localengine/metrics/KVSpaceMetersTest.java | 68 -
.../localengine/rocksdb/KVSpaceSnapshotTest.java | 51 -
.../rocksdb/metrics/KVSpaceMetersTest.java | 81 -
.../bifromq/basekv/raft/BasicStateStoreTest.java | 40 +-
.../bifromq/basekv/raft/ILogEntryIterator.java} | 13 +-
.../org/apache/bifromq/basekv/raft/IRaftNode.java | 16 +-
.../bifromq/basekv/raft/IRaftStateStore.java | 13 +-
.../bifromq/basekv/raft/InMemoryStateStore.java | 29 +-
.../basekv/raft/MetricMonitoredStateStore.java | 13 +-
.../bifromq/basekv/raft/RaftConfigChanger.java | 69 +-
.../org/apache/bifromq/basekv/raft/RaftNode.java | 6 +-
.../apache/bifromq/basekv/raft/RaftNodeState.java | 2 +-
.../bifromq/basekv/raft/RaftNodeStateLeader.java | 66 +-
.../bifromq/basekv/raft/RaftConfigChangerTest.java | 42 +-
.../raft/functest/ChangeClusterConfigTest.java | 157 +-
.../basekv/raft/functest/RaftNodeGroup.java | 53 +-
.../base-kv-split-hinter-spi}/pom.xml | 34 +-
.../basekv/store/range/hinter}/IKVLoadRecord.java | 21 +-
.../store/range/hinter}/IKVRangeSplitHinter.java | 7 +-
.../range/hinter/IKVRangeSplitHinterFactory.java} | 16 +-
.../store/range/hinter/SplitHinterContext.java} | 18 +-
.../basekv/balance/KVStoreBalanceController.java | 21 +
.../impl/RedundantRangeRemovalBalancer.java | 50 +
.../balance/KVStoreBalanceControllerTest.java | 13 +-
.../impl/RedundantRangeRemovalBalancerTest.java | 232 +++
.../basekv/balance/command/CommandType.java | 3 +-
.../basekv/balance/command/MergeCommand.java | 13 +-
.../basekv/balance/command/QuitCommand.java | 25 +-
.../bifromq/basekv/balance/util/CommandUtil.java | 1 +
.../bifromq/basekv/client/BaseKVStoreClient.java | 41 +-
.../bifromq/basekv/client/IBaseKVStoreClient.java | 15 +-
.../basekv/client/ManagedMutationPipeline.java | 11 +-
.../basekv/client/ManagedQueryPipeline.java | 8 +-
.../basekv/client/scheduler/BatchQueryCall.java | 8 +-
.../client/scheduler/BatchQueryCallTest.java | 213 ++-
base-kv/{ => base-kv-store-coproc-api}/pom.xml | 35 +-
.../bifromq/basekv/store/api/IKVIterator.java | 9 +-
.../bifromq/basekv/store/api/IKVRangeCoProc.java | 11 +-
.../basekv/store/api/IKVRangeCoProcFactory.java | 20 +-
.../bifromq/basekv/store/api/IKVRangeReader.java | 112 ++
.../store/api/IKVRangeRefreshableReader.java} | 12 +-
.../apache/bifromq/basekv/store/api/IKVWriter.java | 15 +-
.../org/apache/bifromq/basekv/RPCBluePrint.java | 8 +-
.../src/main/proto/basekv/BaseKVStoreService.proto | 18 +
.../src/main/proto/basekv/Command.proto | 2 +
.../src/main/proto/basekv/StoreMessage.proto | 31 +-
base-kv/base-kv-store-server/pom.xml | 20 +-
.../basekv/server/AgentHostStoreMessenger.java | 46 +-
.../bifromq/basekv/server/BaseKVStoreService.java | 24 +-
.../bifromq/basekv/server/QueryPipeline.java | 16 +-
.../apache/bifromq/basekv/store/IKVRangeStore.java | 23 +-
.../bifromq/basekv/store/KVRangeMessenger.java | 18 +-
.../apache/bifromq/basekv/store/KVRangeStore.java | 342 +++--
.../basekv/store/KVRangeStoreStatsCollector.java | 19 +-
.../basekv/store/api/IKVCloseableReader.java | 25 -
.../apache/bifromq/basekv/store/api/IKVReader.java | 38 -
.../basekv/store/option/KVRangeOptions.java | 4 +-
.../basekv/store/option/KVRangeStoreOptions.java | 70 +-
.../store/range/AbstractKVRangeMetadata.java | 93 --
.../range/AbstractKVRangeMetadataUpdatable.java | 86 --
.../basekv/store/range/IKVCheckpointReader.java | 27 -
.../basekv/store/range/IKVLoadRecorder.java | 8 +-
.../bifromq/basekv/store/range/IKVLoadTracker.java | 69 -
.../bifromq/basekv/store/range/IKVRange.java | 125 +-
.../bifromq/basekv/store/range/IKVRangeFSM.java | 22 +-
.../{IKVReseter.java => IKVRangeIdentifiable.java} | 20 +-
.../store/range/IKVRangeMetadataUpdatable.java | 44 +-
.../basekv/store/range/IKVRangeMetadataWriter.java | 28 -
.../basekv/store/range/IKVRangeMetricManager.java | 12 +-
.../store/range/IKVRangeQueryLinearizer.java | 2 +-
.../IKVRangeRestoreSession.java} | 33 +-
...VReseter.java => IKVRangeSnapshotReceiver.java} | 17 +-
.../basekv/store/range/IKVRangeWritable.java | 27 +-
.../bifromq/basekv/store/range/IKVRangeWriter.java | 2 +-
.../basekv/store/range/KVCheckpointReader.java | 72 -
.../bifromq/basekv/store/range/KVIterator.java | 9 +-
.../bifromq/basekv/store/range/KVLoadRecorder.java | 4 +-
.../apache/bifromq/basekv/store/range/KVRange.java | 304 ++--
.../basekv/store/range/KVRangeCheckpoint.java | 69 -
.../basekv/store/range/KVRangeDumpSession.java | 90 +-
.../bifromq/basekv/store/range/KVRangeFSM.java | 1578 +++++++++++++-------
.../{IKVRangeWritable.java => KVRangeFactory.java} | 18 +-
.../bifromq/basekv/store/range/KVRangeKeys.java | 12 +-
.../basekv/store/range/KVRangeMetricManager.java | 84 +-
.../basekv/store/range/KVRangeQueryLinearizer.java | 48 +-
.../basekv/store/range/KVRangeQueryRunner.java | 35 +-
.../bifromq/basekv/store/range/KVRangeReader.java | 121 ++
...Iterator.java => KVRangeRefreshableReader.java} | 19 +-
...adataWriter.java => KVRangeRestoreSession.java} | 64 +-
.../basekv/store/range/KVRangeRestorer.java | 159 +-
.../store/range/KVRangeSnapshotReceiver.java | 186 +++
.../basekv/store/range/KVRangeStatsCollector.java | 16 +-
.../bifromq/basekv/store/range/KVRangeWriter.java | 131 +-
.../bifromq/basekv/store/range/KVReader.java | 93 --
.../bifromq/basekv/store/range/KVWriter.java | 8 +-
.../store/range/LoadRecordableKVIterator.java | 11 +-
.../store/range/LoadRecordableKVRangeWriter.java | 8 +-
.../basekv/store/range/LoadRecordableKVReader.java | 46 +-
.../basekv/store/range/LoadRecordableKVWriter.java | 2 +-
.../store/range/SnapshotBandwidthGovernor.java | 2 +-
.../store/range/hinter/KVLoadBasedSplitHinter.java | 25 +-
.../store/range/hinter/LoadRecordWindow.java | 30 +-
.../hinter/MutationKVLoadBasedSplitHinter.java | 14 +-
.../MutationKVLoadBasedSplitHinterFactory.java | 26 +-
.../range/hinter/QueryKVLoadBasedSplitHinter.java | 24 +-
.../store/range/hinter/SplitHinterRegistry.java | 64 +
.../bifromq/basekv/store/stats/StatsCollector.java | 11 +-
.../bifromq/basekv/store/wal/IKVRangeWAL.java | 4 +-
.../basekv/store/wal/IKVRangeWALSubscription.java | 6 +-
.../bifromq/basekv/store/wal/KVRangeWAL.java | 16 +-
.../basekv/store/wal/KVRangeWALStorageEngine.java | 12 +-
.../bifromq/basekv/store/wal/KVRangeWALStore.java | 80 +-
.../basekv/store/wal/KVRangeWALSubscription.java | 13 +-
.../bifromq/basekv/store/wal/LogEntryIterator.java | 100 ++
.../basekv/store/wal/LogEntryIteratorPool.java | 138 --
...v.store.range.hinter.IKVRangeSplitHinterFactory | 1 +
.../java/org/apache/bifromq/basekv/TestCoProc.java | 14 +-
.../apache/bifromq/basekv/TestCoProcFactory.java | 6 +-
.../apache/bifromq/basekv/annotation/Cluster.java | 7 +
.../basekv/server/AgentHostStoreMessengerTest.java | 34 +-
.../bifromq/basekv/server/QueryPipelineTest.java | 102 +-
.../bifromq/basekv/store/KVRangeBootstrapTest.java | 24 +-
.../bifromq/basekv/store/KVRangeMessengerTest.java | 68 +-
.../store/KVRangeStoreClusterConfigChangeTest.java | 63 +-
.../KVRangeStoreClusterMergeEdgeCasesTest.java | 777 ++++++++++
.../basekv/store/KVRangeStoreClusterMergeTest.java | 164 +-
.../store/KVRangeStoreClusterTestTemplate.java | 28 +-
.../bifromq/basekv/store/KVRangeStoreTest.java | 84 +-
.../basekv/store/KVRangeStoreTestCluster.java | 410 ++++-
.../basekv/store/range/AbstractKVRangeTest.java | 35 +-
.../store/range/AdaptiveWriteBudgetTest.java | 79 -
.../basekv/store/range/KVRangeDumpSessionTest.java | 113 +-
.../basekv/store/range/KVRangeMetadataTest.java | 43 +-
.../store/range/KVRangeQueryLinearizerTest.java | 22 +-
.../basekv/store/range/KVRangeQueryRunnerTest.java | 85 +-
.../basekv/store/range/KVRangeRestorerTest.java | 198 ++-
.../store/range/KVRangeStatsCollectorTest.java | 23 +-
.../bifromq/basekv/store/range/KVRangeTest.java | 278 +---
.../bifromq/basekv/store/range/KVReaderTest.java | 108 --
.../store/range/hinter/LoadRecorderTest.java | 5 +-
.../hinter/QueryKVIOBasedSplitHinterTest.java | 13 +-
.../range/hinter/RecordingWindowSlotTest.java | 5 +-
.../store/wal/KVRangeWALStoreEngineTest.java | 45 +-
.../basekv/store/wal/KVRangeWALStoreTest.java | 25 +-
.../store/wal/KVRangeWALSubscriptionTest.java | 72 +-
.../bifromq/basekv/store/wal/KVRangeWALTest.java | 6 +-
.../basekv/store/wal/LogEntryIteratorTest.java | 354 +++++
base-kv/pom.xml | 8 +-
.../bifromq/basescheduler/BatchCallScheduler.java | 25 +-
.../basescheduler/BatchCallWeighterFactory.java | 88 ++
.../org/apache/bifromq/basescheduler/Batcher.java | 156 +-
.../basescheduler/CapacityEstimatorFactory.java | 45 +-
.../basescheduler/spi/IBatchCallWeighter.java | 26 +-
...Factory.java => IBatchCallWeighterFactory.java} | 13 +-
.../basescheduler/spi/ICapacityEstimator.java | 34 +-
.../spi/ICapacityEstimatorFactory.java | 12 +-
.../apache/bifromq/basescheduler/BatcherTest.java | 310 ++--
base-util/pom.xml | 15 +
.../org/apache/bifromq/base/util/AsyncRetry.java | 146 +-
.../base/util/CascadeCancelCompletableFuture.java | 186 +++
.../apache/bifromq/base/util/AsyncRetryTest.java | 40 +-
.../apache/bifromq/base/util/AsyncRunnerTest.java | 29 +-
.../util/CascadeCancelCompletableFutureTest.java | 343 +++++
bifromq-bom/pom.xml | 22 +-
.../bifromq/deliverer/BatchDeliveryCall.java | 2 +
.../dist/client/scheduler/BatchPubCall.java | 2 +
.../bifromq/dist/server/DistResponsePipeline.java | 60 +-
.../server/scheduler/TenantRangeLookupCache.java | 74 +-
.../dist/server/DistResponsePipelineTest.java | 154 ++
.../bifromq/dist/server/DistServiceTest.java | 22 +-
.../org/apache/bifromq/dist/worker/DistWorker.java | 4 +-
.../bifromq/dist/worker/DistWorkerBuilder.java | 3 -
.../bifromq/dist/worker/DistWorkerCoProc.java | 258 ++--
.../dist/worker/DistWorkerCoProcFactory.java | 28 +-
.../apache/bifromq/dist/worker/TenantsStats.java | 52 +-
.../dist/worker/cache/SubscriptionCache.java | 4 +-
.../dist/worker/cache/TenantRouteCacheFactory.java | 11 +-
.../dist/worker/cache/TenantRouteMatcher.java | 139 +-
.../dist/worker/cache/ThreadLocalKVReader.java | 19 +-
.../dist/worker/hinter/FanoutSplitHinter.java | 90 +-
.../worker/hinter/FanoutSplitHinterFactory.java | 40 +
...v.store.range.hinter.IKVRangeSplitHinterFactory | 1 +
.../dist/worker/DistWorkerCoProcGCTest.java | 25 +-
.../bifromq/dist/worker/DistWorkerCoProcTest.java | 28 +-
.../apache/bifromq/dist/worker/DistWorkerTest.java | 24 +-
.../bifromq/dist/worker/TenantsStatesTest.java | 9 +-
.../dist/worker/cache/SubscriptionCacheTest.java | 4 -
.../dist/worker/cache/TenantRouteMatcherTest.java | 54 +-
.../dist/worker/cache/ThreadLocalKVReaderTest.java | 21 +-
.../main/proto/inboxservice/InboxStoreCoProc.proto | 18 +-
.../bifromq/inbox/server/FetchSignalSender.java | 44 +
.../bifromq/inbox/server/FetcherSignaler.java | 6 +-
.../apache/bifromq/inbox/server/IInboxFetcher.java | 4 +-
.../bifromq/inbox/server/InboxFetchPipeline.java | 28 +-
.../apache/bifromq/inbox/server/InboxWriter.java | 5 +-
.../bifromq/inbox/server/InboxWriterPipeline.java | 104 +-
.../inbox/server/scheduler/BatchInsertCall.java | 46 +-
.../server/scheduler/InboxCheckSubScheduler.java | 2 +-
.../server/scheduler/InboxExistScheduler.java | 8 +-
.../server/scheduler/InboxFetchScheduler.java | 25 +-
.../server/scheduler/InboxFetchStateScheduler.java | 2 +-
.../inbox/server/scheduler/InboxReadScheduler.java | 10 +-
.../server/scheduler/InboxSendLWTScheduler.java | 6 +-
.../bifromq/inbox/server/InboxExpiryTest.java | 7 +-
.../bifromq/inbox/server/InboxServiceTest.java | 9 +-
.../inbox/server/InboxWriterPipelineTest.java | 58 +-
.../org/apache/bifromq/inbox/store/InboxStore.java | 1 -
.../bifromq/inbox/store/InboxStoreBuilder.java | 1 -
.../bifromq/inbox/store/InboxStoreCoProc.java | 628 +++++---
.../inbox/store/InboxStoreCoProcFactory.java | 36 +-
.../apache/bifromq/inbox/store/TenantStats.java | 2 +-
.../apache/bifromq/inbox/store/TenantsStats.java | 57 +-
.../inbox/store/InboxDeleteClearAllTest.java | 119 ++
.../inbox/store/InboxInsertNewFormatTest.java | 116 ++
.../bifromq/inbox/store/InboxInsertTest.java | 164 +-
.../apache/bifromq/inbox/store/InboxStoreTest.java | 31 +-
.../bifromq/inbox/store/TenantsStatsTest.java | 49 +-
.../org/apache/bifromq/metrics/TenantMetric.java | 5 +
.../bifromq/mqtt/inbox/DeliveryPipeline.java | 19 +-
.../bifromq/mqtt/handler/AdaptiveReceiveQuota.java | 241 +--
.../bifromq/mqtt/handler/IMQTTProtocolHelper.java | 2 +-
.../bifromq/mqtt/handler/MQTTMessageHandler.java | 7 +-
.../mqtt/handler/MQTTPersistentSessionHandler.java | 91 +-
.../bifromq/mqtt/handler/MQTTSessionHandler.java | 106 +-
.../mqtt/handler/MQTTTransientSessionHandler.java | 18 +-
.../apache/bifromq/mqtt/handler/RoutedMessage.java | 2 +-
.../bifromq/mqtt/handler/TenantSettings.java | 5 +-
.../handler/v3/MQTT3PersistentSessionHandler.java | 4 +-
.../mqtt/handler/v3/MQTT3ProtocolHelper.java | 2 +-
.../handler/v3/MQTT3TransientSessionHandler.java | 4 +-
.../mqtt/handler/v5/MQTT5ConnectHandler.java | 13 +
.../bifromq/mqtt/handler/v5/MQTT5MessageUtils.java | 2 +-
.../handler/v5/MQTT5PersistentSessionHandler.java | 4 +-
.../mqtt/handler/v5/MQTT5ProtocolHelper.java | 5 +-
.../handler/v5/MQTT5TransientSessionHandler.java | 4 +-
.../bifromq/mqtt/service/LocalSessionRegistry.java | 6 +-
.../apache/bifromq/mqtt/session/IMQTTSession.java | 4 +-
.../mqtt/handler/AdaptiveReceiveQuotaTest.java | 235 ++-
.../v3/MQTT3PersistentSessionHandlerTest.java | 6 +-
.../v3/MQTT3TransientSessionHandlerTest.java | 47 +-
.../mqtt/handler/v3/MQTTPersistentS2CPubTest.java | 6 +-
.../mqtt/handler/v5/ConnectHandlerTest.java | 32 +
.../handler/v5/PersistentSessionHandlerTest.java | 4 +-
.../v5/PersistentSessionShutdownLWTTest.java | 219 +++
.../handler/v5/TransientSessionHandlerTest.java | 108 +-
.../apache/bifromq/mqtt/integration/MQTTTest.java | 30 +-
.../mqtt/integration/v3/MQTTDisconnectTest.java | 21 +-
.../mqtt/service/LocalSessionRegistryTest.java | 10 +-
.../settingprovider/DevOnlySettingProvider.java | 2 +-
.../bifromq/plugin/settingprovider/Setting.java | 3 +-
.../server/scheduler/MatchCallScheduler.java | 6 +-
.../apache/bifromq/retain/store/RetainStore.java | 2 +-
.../bifromq/retain/store/RetainStoreBuilder.java | 11 +-
.../bifromq/retain/store/RetainStoreCoProc.java | 26 +-
.../retain/store/RetainStoreCoProcFactory.java | 25 +-
.../apache/bifromq/retain/store/TenantStats.java | 23 +-
.../apache/bifromq/retain/store/TenantsStats.java | 70 +-
.../bifromq/retain/store/RetainStoreTest.java | 31 +-
.../bifromq/retain/store/TenantStatsTest.java | 19 +-
.../bifromq/retain/store/TenantsStatsTest.java | 21 +-
.../props/DistWorkerFanOutSplitThreshold.java | 34 -
.../DistWorkerLoadEstimationWindowSeconds.java | 34 -
.../InboxStoreLoadEstimationWindowSeconds.java | 34 -
.../RetainStoreLoadEstimationWindowSeconds.java | 34 -
build/build-bifromq-starter/pom.xml | 6 +-
.../config/StandaloneConfigConsolidator.java | 147 +-
.../bifromq/starter/config/model/EngineConfig.java | 58 +
.../starter/config/model/InMemEngineConfig.java | 23 -
.../starter/config/model/RocksDBEngineConfig.java | 53 -
.../starter/config/model/SplitHinterOptions.java | 33 +-
.../config/model/dist/DistWorkerConfig.java | 45 +-
.../config/model/inbox/InboxStoreClientConfig.java | 2 +-
.../config/model/inbox/InboxStoreConfig.java | 44 +-
.../config/model/retain/RetainStoreConfig.java | 43 +-
.../model/serde/EngineConfigDeserializer.java | 105 ++
.../config/model/serde/EngineConfigSerializer.java | 77 +
.../bifromq/starter/module/CoreServiceModule.java | 10 +-
.../bifromq/starter/module/DistServiceModule.java | 14 +-
.../bifromq/starter/module/EngineConfUtil.java | 118 --
.../bifromq/starter/module/InboxServiceModule.java | 17 +-
.../starter/module/RetainServiceModule.java | 14 +-
.../starter/config/EngineConfigMergeTest.java | 112 ++
coverage-report/pom.xml | 12 +-
447 files changed, 18730 insertions(+), 8759 deletions(-)
diff --git a/.github/workflows/build-cov.yaml b/.github/workflows/build-cov.yaml
index bd0c4b8e6..46ff4e104 100644
--- a/.github/workflows/build-cov.yaml
+++ b/.github/workflows/build-cov.yaml
@@ -9,18 +9,43 @@ on:
branches:
- 'main'
jobs:
- check-license:
- name: "Check License"
+ prepare:
+ name: "Prepare"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: maven
+ - name: Install without tests
+ run: mvn -q -DskipTests install
+
+ license:
+ name: "License Check"
+ needs: prepare
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: maven
- name: Check License Header
uses: apache/skywalking-eyes/header@main
- name: Check Dependencies' License
uses: apache/skywalking-eyes/dependency@main
- build:
+ env:
+ MAVEN_OPTS: -Xmx1g
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ test:
name: "BifroMQ Code Coverage Test"
- needs: check-license
+ needs: license
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/build-dev.yaml b/.github/workflows/build-dev.yaml
index c7bc6a053..5521fd53a 100644
--- a/.github/workflows/build-dev.yaml
+++ b/.github/workflows/build-dev.yaml
@@ -14,18 +14,43 @@ on:
- 'bugfix-**'
- 'fix-**'
jobs:
- check-license:
- name: "Check License"
+ prepare:
+ name: "Prepare"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: maven
+ - name: Install without tests
+ run: mvn -q -DskipTests install
+
+ license:
+ name: "License Check"
+ needs: prepare
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: maven
- name: Check License Header
uses: apache/skywalking-eyes/header@main
- name: Check Dependencies' License
uses: apache/skywalking-eyes/dependency@main
- build:
+ env:
+ MAVEN_OPTS: -Xmx1g
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ test:
name: "Build BifroMQ"
- needs: check-license
+ needs: license
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
diff --git a/.licenserc.yaml b/.licenserc.yaml
index f41f504bb..8ecf5a4d2 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -25,6 +25,9 @@ header:
- '.gitignore'
- '.gitattributes'
- 'lombok.config'
+ - 'META-INF/**'
+ - '**/META-INF/**'
+ - '**/resources/META-INF/**'
- '**/*.crt'
- '**/*.pem'
- '**/MANIFEST.MF'
diff --git a/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/HostMemberList.java b/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/HostMemberList.java
index 25868a209..c2033ef5d 100644
--- a/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/HostMemberList.java
+++ b/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/HostMemberList.java
@@ -381,6 +381,11 @@ public class HostMemberList implements IHostMemberList {
messenger.send(ClusterMessage.newBuilder()
.setJoin(Join.newBuilder().setMember(local).build())
.build(), getMemberAddress(joinMember.getEndpoint()), true);
+ } else if (newMember) {
+ // send back a join to speed up convergence
+ messenger.send(ClusterMessage.newBuilder()
+ .setJoin(Join.newBuilder().setMember(local).build())
+ .build(), getMemberAddress(joinMember.getEndpoint()), true);
}
} else {
clearZombie(join.getExpectedHost());
diff --git a/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/agent/CRDTUtil.java b/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/agent/CRDTUtil.java
index 679d453bb..eab8c8a55 100644
--- a/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/agent/CRDTUtil.java
+++ b/base-cluster/src/main/java/org/apache/bifromq/basecluster/memberlist/agent/CRDTUtil.java
@@ -14,7 +14,7 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basecluster.memberlist.agent;
@@ -23,10 +23,6 @@ import static org.apache.bifromq.basecrdt.core.api.CRDTURI.toURI;
import static org.apache.bifromq.basecrdt.core.api.CausalCRDTType.mvreg;
import static org.apache.bifromq.basecrdt.core.api.CausalCRDTType.ormap;
-import org.apache.bifromq.basecluster.agent.proto.AgentMemberAddr;
-import org.apache.bifromq.basecluster.agent.proto.AgentMemberMetadata;
-import org.apache.bifromq.basecrdt.core.api.IMVReg;
-import org.apache.bifromq.basecrdt.core.api.IORMap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -41,6 +37,10 @@ import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basecluster.agent.proto.AgentMemberAddr;
+import org.apache.bifromq.basecluster.agent.proto.AgentMemberMetadata;
+import org.apache.bifromq.basecrdt.core.api.IMVReg;
+import org.apache.bifromq.basecrdt.core.api.IORMap;
@Slf4j
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@@ -56,7 +56,8 @@ class CRDTUtil {
Iterator<IORMap.ORMapKey> orMapKeyItr = agentCRDT.keys();
while (orMapKeyItr.hasNext()) {
IORMap.ORMapKey orMapKey = orMapKeyItr.next();
- agentMemberMap.put(parseAgentMemberAddr(orMapKey), parseMetadata(agentCRDT.getMVReg(orMapKey.key())).get());
+ Optional<AgentMemberMetadata> meta = parseMetadata(agentCRDT.getMVReg(orMapKey.key()));
+ meta.ifPresent(m -> agentMemberMap.put(parseAgentMemberAddr(orMapKey), m));
}
return agentMemberMap;
}
@@ -78,11 +79,11 @@ class CRDTUtil {
return AgentMemberMetadata.parseFrom(data);
} catch (InvalidProtocolBufferException e) {
log.error("Unable to parse agent host node", e);
- // this exception should not happen
+ // should not happen, skip the broken value
return null;
}
}), Objects::nonNull));
- metaList.sort(Comparator.comparingLong(AgentMemberMetadata::getHlc));
+ metaList.sort(Comparator.comparingLong(AgentMemberMetadata::getHlc).reversed());
return Optional.ofNullable(metaList.isEmpty() ? null : metaList.get(0));
}
}
diff --git a/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/Messenger.java b/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/Messenger.java
index 7a6e6c3ec..7af1d6ecd 100644
--- a/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/Messenger.java
+++ b/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/Messenger.java
@@ -14,17 +14,11 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basecluster.messenger;
-import org.apache.bifromq.basecluster.messenger.proto.DirectMessage;
-import org.apache.bifromq.basecluster.messenger.proto.GossipMessage;
-import org.apache.bifromq.basecluster.messenger.proto.MessengerMessage;
-import org.apache.bifromq.basecluster.proto.ClusterMessage;
-import org.apache.bifromq.basecluster.transport.ITransport;
-import org.apache.bifromq.basecluster.util.RandomUtils;
import com.google.common.collect.Maps;
import com.google.protobuf.InvalidProtocolBufferException;
import io.micrometer.core.instrument.Counter;
@@ -46,14 +40,16 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basecluster.messenger.proto.DirectMessage;
+import org.apache.bifromq.basecluster.messenger.proto.GossipMessage;
+import org.apache.bifromq.basecluster.messenger.proto.MessengerMessage;
+import org.apache.bifromq.basecluster.proto.ClusterMessage;
+import org.apache.bifromq.basecluster.transport.ITransport;
+import org.apache.bifromq.basecluster.util.RandomUtils;
+import org.apache.bifromq.baseenv.ZeroCopyParser;
@Slf4j
public class Messenger implements IMessenger {
- private enum State {
- INIT, START, STOP
- }
-
- private State state = State.INIT;
// threshold for determine which transport to use
private final MessengerTransport transport;
private final CompositeDisposable disposables = new CompositeDisposable();
@@ -61,11 +57,10 @@ public class Messenger implements IMessenger {
private final Gossiper gossiper;
private final Subject<Timed<MessageEnvelope>> publisher =
PublishSubject.<Timed<MessageEnvelope>>create().toSerialized();
-
private final Scheduler scheduler;
private final MessengerOptions opts;
private final MetricManager metricManager;
-
+ private State state = State.INIT;
@Builder
private Messenger(ITransport transport, Scheduler scheduler, MessengerOptions opts) {
this.transport = new MessengerTransport(transport);
@@ -202,8 +197,8 @@ public class Messenger implements IMessenger {
switch (messengerMessageEnvelope.message.getMessengerMessageTypeCase()) {
case DIRECT:
try {
- ClusterMessage clusterMessage =
- ClusterMessage.parseFrom(messengerMessageEnvelope.message.getDirect().getPayload());
+ ClusterMessage clusterMessage = ZeroCopyParser.parse(
+ messengerMessageEnvelope.message.getDirect().getPayload(), ClusterMessage.parser());
log.trace("Received message: sender={}, message={}",
messengerMessageEnvelope.sender, clusterMessage);
metricManager.msgRecvCounters.get(clusterMessage.getClusterMessageTypeCase()).increment();
@@ -244,6 +239,10 @@ public class Messenger implements IMessenger {
log.error("Received unexpected error:", throwable);
}
+ private enum State {
+ INIT, START, STOP
+ }
+
private static class MetricManager {
final Map<ClusterMessage.ClusterMessageTypeCase, Counter> msgSendCounters = Maps.newHashMap();
final Map<ClusterMessage.ClusterMessageTypeCase, Counter> msgRecvCounters = Maps.newHashMap();
@@ -253,17 +252,19 @@ public class Messenger implements IMessenger {
MetricManager(InetSocketAddress localAddress) {
for (ClusterMessage.ClusterMessageTypeCase typeCase : ClusterMessage.ClusterMessageTypeCase.values()) {
- Tags tags = Tags
- .of("local", localAddress.getAddress().getHostAddress() + ":" + localAddress.getPort())
- .and("type", typeCase.name());
- msgSendCounters.put(typeCase,
- Metrics.counter("basecluster.send.count", tags));
- msgRecvCounters.put(typeCase,
- Metrics.counter("basecluster.recv.count", tags));
- gossipGenCounters.put(typeCase,
- Metrics.counter("basecluster.gossip.gen.count", tags));
- gossipHeardCounters.put(typeCase,
- Metrics.counter("basecluster.gossip.heard.count", tags));
+ if (typeCase != ClusterMessage.ClusterMessageTypeCase.CLUSTERMESSAGETYPE_NOT_SET) {
+ Tags tags = Tags
+ .of("local", localAddress.getAddress().getHostAddress() + ":" + localAddress.getPort())
+ .and("type", typeCase.name());
+ msgSendCounters.put(typeCase,
+ Metrics.counter("basecluster.send.count", tags));
+ msgRecvCounters.put(typeCase,
+ Metrics.counter("basecluster.recv.count", tags));
+ gossipGenCounters.put(typeCase,
+ Metrics.counter("basecluster.gossip.gen.count", tags));
+ gossipHeardCounters.put(typeCase,
+ Metrics.counter("basecluster.gossip.heard.count", tags));
+ }
}
}
diff --git a/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/MessengerTransport.java b/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/MessengerTransport.java
index 3bda3c546..0afbfe5d6 100644
--- a/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/MessengerTransport.java
+++ b/base-cluster/src/main/java/org/apache/bifromq/basecluster/messenger/MessengerTransport.java
@@ -33,6 +33,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.bifromq.basecluster.messenger.proto.MessengerMessage;
import org.apache.bifromq.basecluster.transport.ITransport;
import org.apache.bifromq.basecluster.transport.PacketEnvelope;
+import org.apache.bifromq.baseenv.ZeroCopyParser;
@Slf4j
final class MessengerTransport {
@@ -62,9 +63,11 @@ final class MessengerTransport {
private Observable<Timed<MessengerMessageEnvelope>> convert(PacketEnvelope packetEnvelope) {
return Observable.fromIterable(packetEnvelope.data.stream().map(b -> {
try {
+ // Parse with aliasing directly from ByteString to reduce copies
+ MessengerMessage msg = ZeroCopyParser.parse(b, MessengerMessage.parser());
MessengerMessageEnvelope mmEnvelop = MessengerMessageEnvelope.builder()
.recipient(packetEnvelope.recipient)
- .message(MessengerMessage.parseFrom(b))
+ .message(msg)
.sender(packetEnvelope.sender)
.build();
return new Timed<>(mmEnvelop, System.currentTimeMillis(), TimeUnit.MILLISECONDS);
diff --git a/base-cluster/src/main/java/org/apache/bifromq/basecluster/transport/UDPTransport.java b/base-cluster/src/main/java/org/apache/bifromq/basecluster/transport/UDPTransport.java
index 163fdcaf9..c133f1b76 100644
--- a/base-cluster/src/main/java/org/apache/bifromq/basecluster/transport/UDPTransport.java
+++ b/base-cluster/src/main/java/org/apache/bifromq/basecluster/transport/UDPTransport.java
@@ -14,14 +14,11 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basecluster.transport;
-import org.apache.bifromq.basecluster.transport.proto.Packet;
-import org.apache.bifromq.baseenv.NettyEnv;
-import org.apache.bifromq.basehlc.HLC;
import com.google.protobuf.InvalidProtocolBufferException;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
@@ -49,6 +46,10 @@ import java.util.concurrent.TimeUnit;
import lombok.Builder;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basecluster.transport.proto.Packet;
+import org.apache.bifromq.baseenv.NettyEnv;
+import org.apache.bifromq.baseenv.ZeroCopyParser;
+import org.apache.bifromq.basehlc.HLC;
@Slf4j
public final class UDPTransport extends AbstractTransport {
@@ -151,7 +152,8 @@ public final class UDPTransport extends AbstractTransport {
try {
byte[] data = new byte[dp.content().readableBytes()];
dp.content().readBytes(data);
- Packet packet = Packet.parseFrom(data);
+ // Parse with aliasing to avoid extra copies of bytes fields
+ Packet packet = ZeroCopyParser.parse(data, Packet.parser());
transportLatency.record(HLC.INST.getPhysical(packet.getHlc() - HLC.INST.get()));
doReceive(packet, dp.sender(), dp.recipient());
} catch (InvalidProtocolBufferException e) {
diff --git a/base-cluster/src/test/java/org/apache/bifromq/basecluster/memberlist/HostMemberListTest.java b/base-cluster/src/test/java/org/apache/bifromq/basecluster/memberlist/HostMemberListTest.java
index 0a93cf4c2..57e63f5c8 100644
--- a/base-cluster/src/test/java/org/apache/bifromq/basecluster/memberlist/HostMemberListTest.java
+++ b/base-cluster/src/test/java/org/apache/bifromq/basecluster/memberlist/HostMemberListTest.java
@@ -202,6 +202,67 @@ public class HostMemberListTest {
argThat(r -> r.getUri().equals(AGENT_HOST_MAP_URI) && r.getId().equals(LOCAL_STORE_ID)), any());
}
+ @Test
+ public void handleJoinFromNewMemberWithoutExpectedHost() {
+ // ensure local crdt read returns empty and allow address resolve
+ when(hostListCRDT.getMVReg(any())).thenReturn(hostMemberOnCRDT);
+ when(hostMemberOnCRDT.read()).thenReturn(emptyIterator());
+ when(addressResolver.resolve(REMOTE_HOST_1_ENDPOINT)).thenReturn(REMOTE_ADDR_1);
+
+ IHostMemberList memberList = new HostMemberList(LOCAL_ADDR.getHostName(), LOCAL_ADDR.getPort(),
+ messenger, scheduler, store, addressResolver);
+
+ // send a normal join without expectedHost from a new member
+ messageSubject.onNext(joinMsg(HostMember.newBuilder()
+ .setEndpoint(REMOTE_HOST_1_ENDPOINT)
+ .setIncarnation(1)
+ .build()));
+
+ // verify one direct join is sent back to the new member
+ ArgumentCaptor<ClusterMessage> msgCap = ArgumentCaptor.forClass(ClusterMessage.class);
+ ArgumentCaptor<InetSocketAddress> addrCap = ArgumentCaptor.forClass(InetSocketAddress.class);
+ ArgumentCaptor<Boolean> reliableCap = ArgumentCaptor.forClass(Boolean.class);
+ verify(messenger, times(1)).send(msgCap.capture(), addrCap.capture(), reliableCap.capture());
+
+ assertEquals(msgCap.getValue().getJoin().getMember().getEndpoint(), LOCAL_ENDPOINT);
+ assertEquals(msgCap.getValue().getJoin().getMember().getIncarnation(), 0);
+ assertEquals(addrCap.getValue(), REMOTE_ADDR_1);
+ assertTrue(reliableCap.getValue());
+ }
+
+ @Test
+ public void handleDuplicatedJoinWithoutExpectedHost() {
+ // ensure local crdt read returns empty and allow address resolve
+ when(hostListCRDT.getMVReg(any())).thenReturn(hostMemberOnCRDT);
+ when(hostMemberOnCRDT.read()).thenReturn(emptyIterator());
+ when(addressResolver.resolve(REMOTE_HOST_1_ENDPOINT)).thenReturn(REMOTE_ADDR_1);
+
+ IHostMemberList memberList = new HostMemberList(LOCAL_ADDR.getHostName(), LOCAL_ADDR.getPort(),
+ messenger, scheduler, store, addressResolver);
+
+ // first join from new member triggers a send-back
+ messageSubject.onNext(joinMsg(HostMember.newBuilder()
+ .setEndpoint(REMOTE_HOST_1_ENDPOINT)
+ .setIncarnation(1)
+ .build()));
+
+ // duplicated join from the same member should not trigger another send-back
+ messageSubject.onNext(joinMsg(HostMember.newBuilder()
+ .setEndpoint(REMOTE_HOST_1_ENDPOINT)
+ .setIncarnation(1)
+ .build()));
+
+ ArgumentCaptor<ClusterMessage> msgCap = ArgumentCaptor.forClass(ClusterMessage.class);
+ ArgumentCaptor<InetSocketAddress> addrCap = ArgumentCaptor.forClass(InetSocketAddress.class);
+ ArgumentCaptor<Boolean> reliableCap = ArgumentCaptor.forClass(Boolean.class);
+ verify(messenger, times(1)).send(msgCap.capture(), addrCap.capture(), reliableCap.capture());
+
+ assertEquals(msgCap.getValue().getJoin().getMember().getEndpoint(), LOCAL_ENDPOINT);
+ assertEquals(msgCap.getValue().getJoin().getMember().getIncarnation(), 0);
+ assertEquals(addrCap.getValue(), REMOTE_ADDR_1);
+ assertTrue(reliableCap.getValue());
+ }
+
@Test
public void handleJoinAndClearZombie() {
when(hostListCRDT.getMVReg(any())).thenReturn(hostMemberOnCRDT);
diff --git a/base-crdt/base-crdt-service/src/main/java/org/apache/bifromq/basecrdt/service/CRDTCluster.java b/base-crdt/base-crdt-service/src/main/java/org/apache/bifromq/basecrdt/service/CRDTCluster.java
index 25eb0c825..da8bfbdbf 100644
--- a/base-crdt/base-crdt-service/src/main/java/org/apache/bifromq/basecrdt/service/CRDTCluster.java
+++ b/base-crdt/base-crdt-service/src/main/java/org/apache/bifromq/basecrdt/service/CRDTCluster.java
@@ -47,6 +47,7 @@ import org.apache.bifromq.basecrdt.core.api.ICausalCRDT;
import org.apache.bifromq.basecrdt.proto.Replica;
import org.apache.bifromq.basecrdt.store.ICRDTStore;
import org.apache.bifromq.basecrdt.store.proto.CRDTStoreMessage;
+import org.apache.bifromq.baseenv.ZeroCopyParser;
import org.apache.bifromq.logger.MDCLogger;
import org.slf4j.Logger;
@@ -101,7 +102,9 @@ class CRDTCluster<O extends ICRDTOperation, C extends ICausalCRDT<O>> {
return;
}
try {
- this.storeMsgSubject.onNext(CRDTStoreMessage.parseFrom(agentMessage.getPayload()));
+ // Parse with aliasing directly from ByteString
+ this.storeMsgSubject.onNext(
+ ZeroCopyParser.parse(agentMessage.getPayload(), CRDTStoreMessage.parser()));
} catch (InvalidProtocolBufferException e) {
log.error("Unable to parse crdt store message from agent message", e);
}
@@ -113,7 +116,8 @@ class CRDTCluster<O extends ICRDTOperation, C extends ICausalCRDT<O>> {
if (stopped.get()) {
return;
}
- AgentMemberAddr target = AgentMemberAddr.parseFrom(msg.getReceiver());
+ // Parse receiver with aliasing directly from ByteString
+ AgentMemberAddr target = ZeroCopyParser.parse(msg.getReceiver(), AgentMemberAddr.parser()); ;
localMembership.send(target, msg.toByteString(), true)
.whenComplete((v, e) -> {
if (e != null) {
diff --git a/base-env/base-env-provider/src/main/java/org/apache/bifromq/baseenv/ZeroCopyParser.java b/base-env/base-env-provider/src/main/java/org/apache/bifromq/baseenv/ZeroCopyParser.java
index af6cf8e19..9eb649a57 100644
--- a/base-env/base-env-provider/src/main/java/org/apache/bifromq/baseenv/ZeroCopyParser.java
+++ b/base-env/base-env-provider/src/main/java/org/apache/bifromq/baseenv/ZeroCopyParser.java
@@ -14,7 +14,7 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.baseenv;
@@ -23,6 +23,7 @@ import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Parser;
+import com.google.protobuf.UnsafeByteOperations;
/**
* A utility class for parsing protocol buffer messages from a ByteString using zero-copy parsing.
@@ -43,4 +44,17 @@ public class ZeroCopyParser {
input.enableAliasing(true);
return parser.parseFrom(input);
}
+
+ /**
+ * Parses a protocol buffer message from a byte array using zero-copy parsing.
+ *
+ * @param bytes The byte array to parse
+ * @param parser The parser for the protocol buffer message.
+ * @return The parsed protocol buffer message.
+ *
+ * @throws InvalidProtocolBufferException If the parsing fails.
+ */
+ public static <T> T parse(byte[] bytes, Parser<T> parser) throws InvalidProtocolBufferException {
+ return parse(UnsafeByteOperations.unsafeWrap(bytes), parser);
+ }
}
diff --git a/base-kv/base-kv-local-engine/pom.xml b/base-kv/base-kv-local-engine-memory/pom.xml
similarity index 64%
copy from base-kv/base-kv-local-engine/pom.xml
copy to base-kv/base-kv-local-engine-memory/pom.xml
index 398e6036a..c294466c3 100644
--- a/base-kv/base-kv-local-engine/pom.xml
+++ b/base-kv/base-kv-local-engine-memory/pom.xml
@@ -17,7 +17,6 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -27,54 +26,25 @@
<artifactId>base-kv</artifactId>
<version>${revision}</version>
</parent>
-
- <artifactId>base-kv-local-engine</artifactId>
+ <artifactId>base-kv-local-engine-memory</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.bifromq</groupId>
- <artifactId>base-kv-type-proto</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.bifromq</groupId>
- <artifactId>base-env-provider</artifactId>
+ <artifactId>base-kv-local-engine-spi</artifactId>
</dependency>
<dependency>
<groupId>org.apache.bifromq</groupId>
- <artifactId>base-logger</artifactId>
- </dependency>
- <dependency>
- <groupId>io.reactivex.rxjava3</groupId>
- <artifactId>rxjava</artifactId>
- </dependency>
- <dependency>
- <groupId>io.micrometer</groupId>
- <artifactId>micrometer-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <artifactId>base-kv-local-engine-spi</artifactId>
+ <type>test-jar</type>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
- <!-- for testing-->
+ <!-- test -->
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -108,12 +78,4 @@
<scope>test</scope>
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
+</project>
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/AbstractInMemKVSpaceReader.java
similarity index 51%
copy from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
copy to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/AbstractInMemKVSpaceReader.java
index 38b2b53c8..7d71c54b7 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/AbstractInMemKVSpaceReader.java
@@ -19,24 +19,36 @@
package org.apache.bifromq.basekv.localengine.memory;
-import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Optional;
+import org.apache.bifromq.basekv.localengine.AbstractKVSpaceReader;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
-import java.util.concurrent.CompletableFuture;
import org.slf4j.Logger;
-public class InMemWALableKVSpace extends InMemKVSpace<InMemWALableKVEngine, InMemWALableKVSpace>
- implements IWALableKVSpace {
- protected InMemWALableKVSpace(String id,
- InMemKVEngineConfigurator configurator,
- InMemWALableKVEngine engine,
- Runnable onDestroy,
- KVSpaceOpMeters opMeters,
- Logger logger) {
- super(id, configurator, engine, onDestroy, opMeters, logger);
+abstract class AbstractInMemKVSpaceReader extends AbstractKVSpaceReader {
+ protected AbstractInMemKVSpaceReader(String id, KVSpaceOpMeters readOpMeters, Logger logger) {
+ super(id, readOpMeters, logger);
+ }
+
+ protected abstract Map<ByteString, ByteString> metadataMap();
+
+ protected abstract NavigableMap<ByteString, ByteString> rangeData();
+
+ @Override
+ protected Optional<ByteString> doMetadata(ByteString metaKey) {
+ return Optional.ofNullable(metadataMap().get(metaKey));
}
@Override
- public CompletableFuture<Long> flush() {
- return CompletableFuture.completedFuture(System.nanoTime());
+ protected boolean doExist(ByteString key) {
+ return rangeData().containsKey(key);
}
+
+ @Override
+ protected Optional<ByteString> doGet(ByteString key) {
+ return Optional.ofNullable(rangeData().get(key));
+ }
+
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVEngine.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVEngine.java
similarity index 73%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVEngine.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVEngine.java
index 136a7cf6e..7a34557b2 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVEngine.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVEngine.java
@@ -19,21 +19,27 @@
package org.apache.bifromq.basekv.localengine.memory;
+import com.google.protobuf.Struct;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.slf4j.Logger;
-public class InMemCPableKVEngine extends InMemKVEngine<InMemCPableKVEngine, InMemCPableKVSpace> {
- public InMemCPableKVEngine(String overrideIdentity, InMemKVEngineConfigurator c) {
- super(overrideIdentity, c);
+class InMemCPableKVEngine extends InMemKVEngine<InMemCPableKVEngine, InMemCPableKVSpace> {
+ InMemCPableKVEngine(String overrideIdentity, Struct conf) {
+ super(overrideIdentity, conf);
}
@Override
protected InMemCPableKVSpace doBuildKVSpace(String spaceId,
- InMemKVEngineConfigurator configurator,
+ Struct conf,
Runnable onDestroy,
KVSpaceOpMeters opMeters,
Logger logger,
String... tags) {
- return new InMemCPableKVSpace(spaceId, configurator, this, onDestroy, opMeters, logger, tags);
+ return new InMemCPableKVSpace(spaceId, conf, this, onDestroy, opMeters, logger, tags);
+ }
+
+ @Override
+ protected Struct defaultConf() {
+ return InMemDefaultConfigs.CP;
}
}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVSpace.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVSpace.java
new file mode 100644
index 000000000..60368367b
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVSpace.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getGauge;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Tags;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceCheckpoint;
+import org.apache.bifromq.basekv.localengine.IKVSpaceMigratableWriter;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.apache.bifromq.basekv.localengine.RestoreMode;
+import org.apache.bifromq.basekv.localengine.metrics.GeneralKVSpaceMetric;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.slf4j.Logger;
+
+class InMemCPableKVSpace extends InMemKVSpace<InMemCPableKVEngine, InMemCPableKVSpace>
+ implements ICPableKVSpace {
+ private final Cache<String, InMemKVSpaceCheckpoint> checkpoints;
+ private final Gauge checkpointGauge;
+ private final AtomicReference<InMemKVSpaceEpoch> activeEpoch;
+ private volatile InMemKVSpaceCheckpoint latestCheckpoint;
+
+ protected InMemCPableKVSpace(String id,
+ Struct conf,
+ InMemCPableKVEngine engine,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ super(id, conf, engine, onDestroy, opMeters, logger, tags);
+ activeEpoch = new AtomicReference<>(new InMemKVSpaceEpoch());
+ checkpoints = Caffeine.newBuilder().weakValues().build();
+ checkpointGauge = getGauge(id, GeneralKVSpaceMetric.CheckpointNumGauge, checkpoints::estimatedSize,
+ Tags.of(tags));
+ }
+
+ @Override
+ public String checkpoint() {
+ synchronized (this) {
+ return metadataRefresher.call(() -> {
+ String cpId = UUID.randomUUID().toString();
+ latestCheckpoint = new InMemKVSpaceCheckpoint(id, cpId, activeEpoch.get(), opMeters, logger);
+ checkpoints.put(cpId, latestCheckpoint);
+ return cpId;
+ });
+ }
+ }
+
+ @Override
+ public Optional<IKVSpaceCheckpoint> openCheckpoint(String checkpointId) {
+ return Optional.ofNullable(checkpoints.getIfPresent(checkpointId));
+ }
+
+ @Override
+ public IRestoreSession startRestore(IRestoreSession.FlushListener flushListener) {
+ return new RestoreSession(RestoreMode.Replace, flushListener);
+ }
+
+ @Override
+ public IRestoreSession startReceiving(IRestoreSession.FlushListener flushListener) {
+ return new RestoreSession(RestoreMode.Overlay, flushListener);
+ }
+
+ @Override
+ protected void doClose() {
+ checkpointGauge.close();
+ }
+
+ @Override
+ protected void doDestroy() {
+ activeEpoch.set(new InMemKVSpaceEpoch());
+ }
+
+ @Override
+ protected InMemKVSpaceEpoch handle() {
+ return activeEpoch.get();
+ }
+
+ @Override
+ protected void doOpen() {
+
+ }
+
+ @Override
+ public IKVSpaceMigratableWriter toWriter() {
+ return new InMemKVSpaceMigratableWriter<>(id, activeEpoch.get(), engine, syncContext, metadataUpdated -> {
+ if (metadataUpdated) {
+ this.loadMetadata();
+ }
+ }, impact -> {
+ // Update tracked boundary sizes on write impact
+ this.tracker.updateOnWrite(impact, activeEpoch.get().dataMap());
+ }, opMeters, logger);
+ }
+
+ private class RestoreSession implements IRestoreSession {
+ private final InMemKVSpaceEpoch staging;
+ private final IRestoreSession.FlushListener flushListener;
+ private final AtomicBoolean closed = new AtomicBoolean();
+ private int ops = 0;
+ private long bytes = 0;
+
+ private RestoreSession(RestoreMode mode, FlushListener flushListener) {
+ switch (mode) {
+ case Overlay -> staging = new InMemKVSpaceEpoch(activeEpoch.get());
+ case Replace -> staging = new InMemKVSpaceEpoch();
+ default -> throw new IllegalArgumentException("Unsupported restore mode: " + mode);
+ }
+ this.flushListener = flushListener;
+ }
+
+ private void ensureOpen() {
+ if (closed.get()) {
+ throw new IllegalStateException("Restore session already closed");
+ }
+ }
+
+ @Override
+ public IRestoreSession put(ByteString key, ByteString value) {
+ ensureOpen();
+ staging.putData(key, value);
+ ops++;
+ bytes += key.size() + value.size();
+ return this;
+ }
+
+ @Override
+ public IRestoreSession metadata(ByteString metaKey, ByteString metaValue) {
+ ensureOpen();
+ staging.setMetadata(metaKey, metaValue);
+ ops++;
+ bytes += metaKey.size() + metaValue.size();
+ return this;
+ }
+
+ @Override
+ public void done() {
+ ensureOpen();
+ if (closed.compareAndSet(false, true)) {
+ // Replace mode ignores existing state; Overlay mode applies on top of current state
+ syncContext().mutator().run(() -> {
+ activeEpoch.set(staging);
+ // Epoch changed, invalidate tracked boundary sizes
+ tracker.invalidateAll();
+ if (flushListener != null) {
+ flushListener.onFlush(ops, bytes);
+ }
+ loadMetadata();
+ // InMemCPableKVSpace is not generation aware
+ return true;
+ });
+ }
+ }
+
+ @Override
+ public void abort() {
+ if (closed.compareAndSet(false, true)) {
+ // no-op
+ }
+ }
+
+ @Override
+ public int count() {
+ return ops;
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/KVSpaceDescriptor.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemDefaultConfigs.java
similarity index 70%
copy from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/KVSpaceDescriptor.java
copy to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemDefaultConfigs.java
index b7ae63080..7f9a36fa4 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/KVSpaceDescriptor.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemDefaultConfigs.java
@@ -17,15 +17,17 @@
* under the License.
*/
-package org.apache.bifromq.basekv.localengine;
+package org.apache.bifromq.basekv.localengine.memory;
-import java.util.Map;
+import com.google.protobuf.Struct;
/**
- * Descriptor record of a KV space.
- *
- * @param id the id of the space
- * @param metrics the metrics of the space
+ * Default configuration constants for in-memory engines.
*/
-public record KVSpaceDescriptor(String id, Map<String, Double> metrics) {
+public final class InMemDefaultConfigs {
+ public static final Struct CP = Struct.newBuilder().build();
+ public static final Struct WAL = CP;
+
+ private InMemDefaultConfigs() {
+ }
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngine.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngine.java
similarity index 89%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngine.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngine.java
index b5d77ad97..9a03c2974 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngine.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngine.java
@@ -19,15 +19,16 @@
package org.apache.bifromq.basekv.localengine.memory;
-import org.apache.bifromq.basekv.localengine.AbstractKVEngine;
+import com.google.protobuf.Struct;
import java.util.UUID;
+import org.apache.bifromq.basekv.localengine.AbstractKVEngine;
abstract class InMemKVEngine<E extends InMemKVEngine<E, T>, T extends InMemKVSpace<E, T>>
- extends AbstractKVEngine<T, InMemKVEngineConfigurator> {
+ extends AbstractKVEngine<T> {
private final String identity;
- public InMemKVEngine(String overrideIdentity, InMemKVEngineConfigurator c) {
- super(overrideIdentity, c);
+ public InMemKVEngine(String overrideIdentity, Struct conf) {
+ super(overrideIdentity, conf);
if (overrideIdentity != null && !overrideIdentity.trim().isEmpty()) {
identity = overrideIdentity;
} else {
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineProvider.java
similarity index 52%
copy from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
copy to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineProvider.java
index 28cb205bc..b66d3c798 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineProvider.java
@@ -19,13 +19,39 @@
package org.apache.bifromq.basekv.localengine.memory;
-import org.apache.bifromq.basekv.localengine.AbstractKVEngineTest;
+import com.google.protobuf.Struct;
import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
+import org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider;
+
+/**
+ * Provider for in-memory engine implementation.
+ */
+public class InMemKVEngineProvider implements IKVEngineProvider {
+
+ @Override
+ public String type() {
+ return "memory";
+ }
+
+ @Override
+ public IKVEngine<? extends ICPableKVSpace> createCPable(String overrideIdentity, Struct conf) {
+ return new InMemCPableKVEngine(overrideIdentity, conf);
+ }
+
+ @Override
+ public IKVEngine<? extends IWALableKVSpace> createWALable(String overrideIdentity, Struct conf) {
+ return new InMemWALableKVEngine(overrideIdentity, conf);
+ }
+
+ @Override
+ public Struct defaultsForCPable() {
+ return InMemDefaultConfigs.CP;
+ }
-public class InMemKVEngineTest extends AbstractKVEngineTest {
@Override
- protected IKVEngine<? extends ICPableKVSpace> newEngine() {
- return new InMemCPableKVEngine(null, new InMemKVEngineConfigurator());
+ public Struct defaultsForWALable() {
+ return InMemDefaultConfigs.WAL;
}
}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVHelper.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVHelper.java
new file mode 100644
index 000000000..cc23decad
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVHelper.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.SortedMap;
+import org.apache.bifromq.basekv.proto.Boundary;
+
+public class InMemKVHelper {
+ public static long sizeOfRange(NavigableMap<ByteString, ByteString> rangeData, Boundary boundary) {
+ SortedMap<ByteString, ByteString> sizedData;
+ if (!boundary.hasStartKey() && !boundary.hasEndKey()) {
+ sizedData = rangeData;
+ } else if (!boundary.hasStartKey()) {
+ sizedData = rangeData.headMap(boundary.getEndKey());
+ } else if (!boundary.hasEndKey()) {
+ sizedData = rangeData.tailMap(boundary.getStartKey());
+ } else {
+ sizedData = rangeData.subMap(boundary.getStartKey(), boundary.getEndKey());
+ }
+ long sum = 0L;
+ for (Map.Entry<ByteString, ByteString> e : sizedData.entrySet()) {
+ sum += e.getKey().size() + e.getValue().size();
+ }
+ return sum;
+ }
+}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpace.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpace.java
new file mode 100644
index 000000000..1f7057e6a
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpace.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import java.util.Collections;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import org.apache.bifromq.basekv.localengine.AbstractKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.SyncContext;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.apache.bifromq.basekv.utils.BoundaryUtil;
+import org.slf4j.Logger;
+
+abstract class InMemKVSpace<
+ E extends InMemKVEngine<E, T>,
+ T extends InMemKVSpace<E, T>> extends AbstractKVSpace<InMemKVSpaceEpoch> {
+ protected final E engine;
+ protected final ISyncContext syncContext = new SyncContext();
+ protected final ISyncContext.IRefresher metadataRefresher = syncContext.refresher();
+ protected final TrackedBoundaryIndex tracker = new TrackedBoundaryIndex();
+
+ protected InMemKVSpace(String id,
+ Struct conf,
+ E engine,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ super(id, onDestroy, opMeters, logger, tags);
+ this.engine = engine;
+ }
+
+ ISyncContext syncContext() {
+ return syncContext;
+ }
+
+ @Override
+ public IKVSpaceRefreshableReader reader() {
+ return new InMemKVSpaceReader(id, opMeters, logger, syncContext.refresher(), this::handle, tracker);
+ }
+
+ protected void loadMetadata() {
+ metadataRefresher.runIfNeeded((genBumped) -> {
+ if (!handle().metadataMap().isEmpty()) {
+ updateMetadata(Collections.unmodifiableMap(handle().metadataMap()));
+ }
+ });
+ }
+
+ protected long doSize(Boundary boundary) {
+ if (!boundary.hasStartKey() && !boundary.hasEndKey()) {
+ return handle().totalDataBytes();
+ }
+ // Track boundary size lazily and keep it updated on writes
+ return tracker.sizeOrTrack(handle(), boundary);
+ }
+
+ static final class TrackedBoundaryIndex {
+ private static final int MAX_TRACKED = 1024;
+ private final NavigableMap<Boundary, TrackedBucket> buckets =
+ new ConcurrentSkipListMap<>(BoundaryUtil::compare);
+
+ long sizeOrTrack(InMemKVSpaceEpoch epoch, Boundary boundary) {
+ TrackedBucket b = buckets.get(boundary);
+ if (b != null) {
+ b.touch();
+ return b.bytes;
+ }
+ long sized = InMemKVHelper.sizeOfRange(epoch.dataMap(), boundary);
+ if (buckets.size() >= MAX_TRACKED) {
+ buckets.pollFirstEntry();
+ }
+ buckets.put(boundary, new TrackedBucket(sized));
+ return sized;
+ }
+
+ void updateOnWrite(InMemKVSpaceWriterHelper.WriteImpact impact, NavigableMap<ByteString, ByteString> data) {
+ if (impact == null || buckets.isEmpty()) {
+ return;
+ }
+ Map<ByteString, Integer> delta = impact.pointDeltaBytes();
+ if (delta == null || delta.isEmpty()) {
+ return;
+ }
+ final long now = System.nanoTime();
+ for (Map.Entry<Boundary, TrackedBucket> e : buckets.entrySet()) {
+ final Boundary tracked = e.getKey();
+ final TrackedBucket bucket = e.getValue();
+ long deltaSum = 0L;
+ // accumulate per-key delta inside the boundary
+ for (Map.Entry<ByteString, Integer> de : delta.entrySet()) {
+ if (BoundaryUtil.inRange(de.getKey(), tracked)) {
+ deltaSum += de.getValue();
+ }
+ }
+ if (deltaSum != 0L) {
+ long newBytes = bucket.bytes + deltaSum;
+ bucket.bytes = Math.max(newBytes, 0L);
+ bucket.lastAccessNanos = now;
+ }
+ }
+ }
+
+ void invalidateAll() {
+ buckets.clear();
+ }
+
+ private static final class TrackedBucket {
+ volatile long bytes;
+ volatile long lastAccessNanos;
+
+ TrackedBucket(long bytes) {
+ this.bytes = bytes;
+ this.lastAccessNanos = System.nanoTime();
+ }
+
+ void touch() {
+ lastAccessNanos = System.nanoTime();
+ }
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpoint.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpoint.java
similarity index 61%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpoint.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpoint.java
index 8e910c16d..4d9d9a3a9 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpoint.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpoint.java
@@ -14,33 +14,33 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basekv.localengine.memory;
import org.apache.bifromq.basekv.localengine.IKVSpaceCheckpoint;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
-import com.google.protobuf.ByteString;
-import java.util.Map;
-import java.util.concurrent.ConcurrentSkipListMap;
import org.slf4j.Logger;
-class InMemKVSpaceCheckpoint extends InMemKVSpaceReader implements IKVSpaceCheckpoint {
+class InMemKVSpaceCheckpoint implements IKVSpaceCheckpoint {
+ private final String id;
+ private final KVSpaceOpMeters opMeters;
+ private final Logger logger;
private final String cpId;
- private final Map<ByteString, ByteString> metadataMap;
- private final ConcurrentSkipListMap<ByteString, ByteString> rangeData;
+ private final InMemKVSpaceEpoch checkpoint;
protected InMemKVSpaceCheckpoint(String id,
String cpId,
- Map<ByteString, ByteString> metadataMap,
- ConcurrentSkipListMap<ByteString, ByteString> rangeData,
+ InMemKVSpaceEpoch checkpoint,
KVSpaceOpMeters opMeters,
Logger logger) {
- super(id, opMeters, logger);
+ this.id = id;
+ this.opMeters = opMeters;
+ this.logger = logger;
this.cpId = cpId;
- this.metadataMap = metadataMap;
- this.rangeData = rangeData;
+ this.checkpoint = checkpoint;
}
@Override
@@ -49,12 +49,7 @@ class InMemKVSpaceCheckpoint extends InMemKVSpaceReader implements IKVSpaceCheck
}
@Override
- protected Map<ByteString, ByteString> metadataMap() {
- return metadataMap;
- }
-
- @Override
- protected ConcurrentSkipListMap<ByteString, ByteString> rangeData() {
- return rangeData;
+ public IKVSpaceReader newReader() {
+ return new InMemKVSpaceCheckpointReader(id, opMeters, logger, checkpoint);
}
}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpointReader.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpointReader.java
new file mode 100644
index 000000000..8c56c4770
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceCheckpointReader.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import static org.apache.bifromq.basekv.localengine.memory.InMemKVHelper.sizeOfRange;
+
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.NavigableMap;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.slf4j.Logger;
+
+class InMemKVSpaceCheckpointReader extends AbstractInMemKVSpaceReader {
+ private final InMemKVSpaceEpoch checkpoint;
+
+ protected InMemKVSpaceCheckpointReader(String id,
+ KVSpaceOpMeters readOpMeters,
+ Logger logger,
+ InMemKVSpaceEpoch checkpoint) {
+ super(id, readOpMeters, logger);
+ this.checkpoint = checkpoint;
+ }
+
+ @Override
+ protected Map<ByteString, ByteString> metadataMap() {
+ return checkpoint.metadataMap();
+ }
+
+ @Override
+ protected NavigableMap<ByteString, ByteString> rangeData() {
+ return checkpoint.dataMap();
+ }
+
+ @Override
+ protected IKVSpaceIterator doNewIterator(Boundary subBoundary) {
+ return new InMemKVSpaceIterator(rangeData(), subBoundary);
+ }
+
+ @Override
+ protected long doSize(Boundary boundary) {
+ return sizeOfRange(checkpoint.dataMap(), boundary);
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceEpoch.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceEpoch.java
new file mode 100644
index 000000000..e472d9a32
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceEpoch.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import static com.google.protobuf.ByteString.unsignedLexicographicalComparator;
+
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bifromq.basekv.localengine.IKVSpaceEpoch;
+import org.pcollections.HashPMap;
+import org.pcollections.HashTreePMap;
+import org.pcollections.TreePMap;
+
+class InMemKVSpaceEpoch implements IKVSpaceEpoch {
+ private final AtomicReference<HashPMap<ByteString, ByteString>> metadataMap;
+ private final AtomicReference<TreePMap<ByteString, ByteString>> dataMap;
+ private final AtomicLong totalDataBytes;
+
+ InMemKVSpaceEpoch() {
+ metadataMap = new AtomicReference<>(HashTreePMap.empty());
+ dataMap = new AtomicReference<>(TreePMap.empty(unsignedLexicographicalComparator()));
+ totalDataBytes = new AtomicLong(0);
+ }
+
+ InMemKVSpaceEpoch(InMemKVSpaceEpoch overlay) {
+ metadataMap = new AtomicReference<>(overlay.metadataMap.get());
+ dataMap = new AtomicReference<>(overlay.dataMap.get());
+ totalDataBytes = new AtomicLong(overlay.totalDataBytes.get());
+ }
+
+ Map<ByteString, ByteString> metadataMap() {
+ return metadataMap.get();
+ }
+
+ NavigableMap<ByteString, ByteString> dataMap() {
+ return dataMap.get();
+ }
+
+ long totalDataBytes() {
+ return totalDataBytes.get();
+ }
+
+ void setMetadata(ByteString key, ByteString value) {
+ metadataMap.updateAndGet(m -> m.plus(key, value));
+ }
+
+ void removeMetadata(ByteString key) {
+ metadataMap.updateAndGet(m -> m.minus(key));
+ }
+
+ void putData(ByteString key, ByteString value) {
+ TreePMap<ByteString, ByteString> current = dataMap.get();
+ ByteString old = current.get(key);
+ long oldBytes = old == null ? 0 : (long) key.size() + old.size();
+ long newBytes = (long) key.size() + value.size();
+ totalDataBytes.addAndGet(newBytes - oldBytes);
+ dataMap.set(current.plus(key, value));
+ }
+
+ void removeData(ByteString key) {
+ TreePMap<ByteString, ByteString> current = dataMap.get();
+ ByteString old = current.get(key);
+ if (old != null) {
+ totalDataBytes.addAndGet(-((long) key.size() + old.size()));
+ dataMap.set(current.minus(key));
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceIterator.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceIterator.java
similarity index 51%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceIterator.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceIterator.java
index 2a9217c1d..65ffbd386 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceIterator.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceIterator.java
@@ -14,32 +14,36 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basekv.localengine.memory;
-import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
-import org.apache.bifromq.basekv.proto.Boundary;
import com.google.protobuf.ByteString;
import java.util.Map;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.NavigableMap;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.proto.Boundary;
class InMemKVSpaceIterator implements IKVSpaceIterator {
- private final ConcurrentSkipListMap<ByteString, ByteString> origData;
private final Boundary boundary;
+ private final AtomicReference<NavigableMap<ByteString, ByteString>> currentEpoch = new AtomicReference<>();
+ private final CloseListener closeListener;
private Map.Entry<ByteString, ByteString> currentEntry;
- private ConcurrentNavigableMap<ByteString, ByteString> dataSource;
+ private NavigableMap<ByteString, ByteString> dataSource;
- public InMemKVSpaceIterator(ConcurrentSkipListMap<ByteString, ByteString> data) {
- this(data, Boundary.getDefaultInstance());
+ public InMemKVSpaceIterator(NavigableMap<ByteString, ByteString> epoch, Boundary boundary) {
+ this(epoch, boundary, iterator -> {
+ });
}
- public InMemKVSpaceIterator(ConcurrentSkipListMap<ByteString, ByteString> data, Boundary boundary) {
- origData = data;
+ public InMemKVSpaceIterator(NavigableMap<ByteString, ByteString> epoch,
+ Boundary boundary,
+ CloseListener closeListener) {
this.boundary = boundary;
- refresh();
+ this.closeListener = closeListener;
+ refresh(epoch);
}
@Override
@@ -59,51 +63,68 @@ class InMemKVSpaceIterator implements IKVSpaceIterator {
@Override
public void next() {
- currentEntry = dataSource.higherEntry(currentEntry.getKey());
+ currentEntry = dataSource().higherEntry(currentEntry.getKey());
}
@Override
public void prev() {
- currentEntry = dataSource.lowerEntry(currentEntry.getKey());
+ currentEntry = dataSource().lowerEntry(currentEntry.getKey());
}
@Override
public void seekToFirst() {
- currentEntry = dataSource.firstEntry();
+ if (dataSource().isEmpty()) {
+ currentEntry = null;
+ return;
+ }
+ currentEntry = dataSource().firstEntry();
}
@Override
public void seekToLast() {
- currentEntry = dataSource.lastEntry();
+ if (dataSource().isEmpty()) {
+ currentEntry = null;
+ return;
+ }
+ currentEntry = dataSource().lastEntry();
}
@Override
public void seek(ByteString target) {
- currentEntry = dataSource.ceilingEntry(target);
+ currentEntry = dataSource().ceilingEntry(target);
}
@Override
public void seekForPrev(ByteString target) {
- currentEntry = dataSource.floorEntry(target);
+ currentEntry = dataSource().floorEntry(target);
}
- @Override
- public void refresh() {
- ConcurrentSkipListMap<ByteString, ByteString> data = origData.clone();
+ public void refresh(NavigableMap<ByteString, ByteString> epoch) {
+ currentEntry = null;
+ currentEpoch.set(epoch);
+ NavigableMap<ByteString, ByteString> data = currentEpoch.get();
if (!boundary.hasStartKey() && !boundary.hasEndKey()) {
dataSource = data;
} else if (!boundary.hasStartKey()) {
- dataSource = data.headMap(boundary.getEndKey());
+ dataSource = data.headMap(boundary.getEndKey(), false);
} else if (!boundary.hasEndKey()) {
- dataSource = data.tailMap(boundary.getStartKey());
+ dataSource = data.tailMap(boundary.getStartKey(), true);
} else {
- dataSource = data.subMap(boundary.getStartKey(), boundary.getEndKey());
+ dataSource = data.subMap(boundary.getStartKey(), true, boundary.getEndKey(), false);
}
- currentEntry = dataSource.firstEntry();
+ }
+
+ private NavigableMap<ByteString, ByteString> dataSource() {
+ return dataSource;
}
@Override
public void close() {
currentEntry = null;
+ closeListener.onClose(this);
+ }
+
+ interface CloseListener {
+ void onClose(InMemKVSpaceIterator iterator);
}
}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceMigratableWriter.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceMigratableWriter.java
new file mode 100644
index 000000000..06992c063
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceMigratableWriter.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import java.util.function.Consumer;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceMigratableWriter;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.slf4j.Logger;
+
+public class InMemKVSpaceMigratableWriter<E extends InMemKVEngine<E, T>, T extends InMemKVSpace<E, T>>
+ extends InMemKVSpaceWriter<E, T> implements IKVSpaceMigratableWriter {
+
+ InMemKVSpaceMigratableWriter(String id,
+ InMemKVSpaceEpoch epoch,
+ E engine,
+ ISyncContext syncContext,
+ Consumer<Boolean> afterWrite,
+ Consumer<InMemKVSpaceWriterHelper.WriteImpact> impactListener,
+ KVSpaceOpMeters readOpMeters,
+ Logger logger) {
+ super(id, epoch, engine, syncContext, afterWrite, impactListener, readOpMeters, logger);
+ }
+
+ @Override
+ public IRestoreSession migrateTo(String targetSpaceId, Boundary boundary) {
+ try {
+ InMemCPableKVSpace targetKVSpace = (InMemCPableKVSpace) engine.createIfMissing(targetSpaceId);
+ IRestoreSession session = targetKVSpace.startRestore(((count, bytes) ->
+ logger.debug("Migrate {} kv to space[{}] from space[{}]: startKey={}, endKey={}",
+ count, targetSpaceId, id, boundary.getStartKey().toStringUtf8(),
+ boundary.getEndKey().toStringUtf8())));
+ // move data
+ try (IKVSpaceIterator itr = new InMemKVSpaceIterator(epoch.dataMap(), boundary)) {
+ for (itr.seekToFirst(); itr.isValid(); itr.next()) {
+ session.put(itr.key(), itr.value());
+ }
+ }
+ // clear moved data in left range
+ helper.clear(id, boundary);
+ return session;
+ } catch (Throwable e) {
+ throw new KVEngineException("Delete range in batch failed", e);
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceReader.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceReader.java
new file mode 100644
index 000000000..fbddd6e34
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceReader.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import com.google.common.collect.Sets;
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.function.Supplier;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.apache.bifromq.basekv.utils.BoundaryUtil;
+import org.slf4j.Logger;
+
+class InMemKVSpaceReader extends AbstractInMemKVSpaceReader implements IKVSpaceRefreshableReader {
+ private static final int MAX_CACHE_ENTRIES = 1024;
+ private final ISyncContext.IRefresher refresher;
+ private final Supplier<InMemKVSpaceEpoch> epochSupplier;
+ private final InMemKVSpace.TrackedBoundaryIndex tracker;
+ private final Set<InMemKVSpaceIterator> openedIterators = Sets.newConcurrentHashSet();
+ private final Map<Boundary, Long> sizeCache = new ConcurrentSkipListMap<>(BoundaryUtil::compare);
+ private volatile InMemKVSpaceEpoch currentEpoch;
+
+ InMemKVSpaceReader(String id,
+ KVSpaceOpMeters readOpMeters,
+ Logger logger,
+ ISyncContext.IRefresher refresher,
+ Supplier<InMemKVSpaceEpoch> epochSupplier,
+ InMemKVSpace.TrackedBoundaryIndex tracker) {
+ super(id, readOpMeters, logger);
+ this.refresher = refresher;
+ this.epochSupplier = epochSupplier;
+ this.tracker = tracker;
+ this.currentEpoch = epochSupplier.get();
+ }
+
+ @Override
+ protected Map<ByteString, ByteString> metadataMap() {
+ return currentEpoch.metadataMap();
+ }
+
+ @Override
+ protected NavigableMap<ByteString, ByteString> rangeData() {
+ return currentEpoch.dataMap();
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void refresh() {
+ refresher.runIfNeeded((genBumped) -> {
+ currentEpoch = epochSupplier.get();
+ sizeCache.clear();
+ openedIterators.forEach(itr -> itr.refresh(currentEpoch.dataMap()));
+ });
+ }
+
+ @Override
+ protected IKVSpaceIterator doNewIterator(Boundary subBoundary) {
+ InMemKVSpaceIterator itr = new InMemKVSpaceIterator(rangeData(), subBoundary, openedIterators::remove);
+ openedIterators.add(itr);
+ return itr;
+ }
+
+ @Override
+ protected long doSize(Boundary boundary) {
+ // Fast path for full-range size
+ if (!boundary.hasStartKey() && !boundary.hasEndKey()) {
+ return currentEpoch.totalDataBytes();
+ }
+ // Consult reader-local cache first
+ Long cached = sizeCache.get(boundary);
+ if (cached != null) {
+ return cached;
+ }
+ // Delegate to space-level tracker for lazy tracking
+ long sized = tracker.sizeOrTrack(currentEpoch, boundary);
+ if (sizeCache.size() >= MAX_CACHE_ENTRIES) {
+ try {
+ @SuppressWarnings("unchecked")
+ ConcurrentSkipListMap<Boundary, Long> sl = (ConcurrentSkipListMap<Boundary, Long>) sizeCache;
+ if (sl.pollFirstEntry() == null && !sizeCache.isEmpty()) {
+ // fallback best-effort
+ sizeCache.remove(sizeCache.keySet().iterator().next());
+ }
+ } catch (ClassCastException ignored) {
+ // best-effort fallback if type changes in future
+ if (!sizeCache.isEmpty()) {
+ sizeCache.remove(sizeCache.keySet().iterator().next());
+ }
+ }
+ }
+ sizeCache.put(boundary, sized);
+ return sized;
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriter.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriter.java
similarity index 54%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriter.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriter.java
index a22448df2..ac8236a2c 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriter.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriter.java
@@ -19,56 +19,61 @@
package org.apache.bifromq.basekv.localengine.memory;
-import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
-import org.apache.bifromq.basekv.localengine.IKVSpaceMetadataWriter;
+import com.google.protobuf.ByteString;
+import java.util.function.Consumer;
import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
import org.apache.bifromq.basekv.localengine.ISyncContext;
import org.apache.bifromq.basekv.localengine.KVEngineException;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.apache.bifromq.basekv.proto.Boundary;
-import com.google.protobuf.ByteString;
-import java.util.Map;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.function.Consumer;
import org.slf4j.Logger;
-class InMemKVSpaceWriter<E extends InMemKVEngine<E, T>, T extends InMemKVSpace<E, T>> extends InMemKVSpaceReader
- implements IKVSpaceWriter {
- private final Map<ByteString, ByteString> metadataMap;
- private final ConcurrentSkipListMap<ByteString, ByteString> rangeData;
- private final E engine;
- private final InMemKVSpaceWriterHelper helper;
+class InMemKVSpaceWriter<E extends InMemKVEngine<E, T>, T extends InMemKVSpace<E, T>> implements IKVSpaceWriter {
+ protected final String id;
+ protected final KVSpaceOpMeters opMeters;
+ protected final Logger logger;
+ protected final InMemKVSpaceWriterHelper helper;
+ protected final E engine;
+ protected final InMemKVSpaceEpoch epoch;
InMemKVSpaceWriter(String id,
- Map<ByteString, ByteString> metadataMap,
- ConcurrentSkipListMap<ByteString, ByteString> rangeData,
+ InMemKVSpaceEpoch epoch,
E engine,
ISyncContext syncContext,
Consumer<Boolean> afterWrite,
+ Consumer<InMemKVSpaceWriterHelper.WriteImpact> impactListener,
KVSpaceOpMeters readOpMeters,
Logger logger) {
- this(id, metadataMap, rangeData, engine, syncContext, new InMemKVSpaceWriterHelper(),
- afterWrite, readOpMeters, logger);
+ this(id, epoch, engine, syncContext, new InMemKVSpaceWriterHelper(),
+ afterWrite, impactListener, readOpMeters, logger);
}
private InMemKVSpaceWriter(String id,
- Map<ByteString, ByteString> metadataMap,
- ConcurrentSkipListMap<ByteString, ByteString> rangeData,
+ InMemKVSpaceEpoch epoch,
E engine,
ISyncContext syncContext,
InMemKVSpaceWriterHelper writerHelper,
Consumer<Boolean> afterWrite,
+ Consumer<InMemKVSpaceWriterHelper.WriteImpact> impactListener,
KVSpaceOpMeters readOpMeters,
Logger logger) {
- super(id, readOpMeters, logger);
- this.metadataMap = metadataMap;
- this.rangeData = rangeData;
+ this.id = id;
+ this.opMeters = readOpMeters;
+ this.logger = logger;
+ this.epoch = epoch;
this.engine = engine;
this.helper = writerHelper;
- writerHelper.addMutators(id, metadataMap, rangeData, syncContext.mutator());
+ writerHelper.addMutators(id, epoch, syncContext.mutator());
writerHelper.addAfterWriteCallback(id, afterWrite);
+ if (impactListener != null) {
+ writerHelper.addAfterImpactCallback(id, impactListener);
+ }
}
+ @Override
+ public String id() {
+ return id;
+ }
@Override
public IKVSpaceWriter metadata(ByteString metaKey, ByteString metaValue) {
@@ -106,45 +111,6 @@ class InMemKVSpaceWriter<E extends InMemKVEngine<E, T>, T extends InMemKVSpace<E
return this;
}
- @Override
- public IKVSpaceMetadataWriter migrateTo(String targetSpaceId, Boundary boundary) {
- try {
- InMemKVSpace<?, ?> targetKVSpace = engine.createIfMissing(targetSpaceId);
- IKVSpaceWriter targetKVSpaceWriter = targetKVSpace.toWriter();
- // move data
- try (IKVSpaceIterator itr = newIterator(boundary)) {
- for (itr.seekToFirst(); itr.isValid(); itr.next()) {
- targetKVSpaceWriter.put(itr.key(), itr.value());
- }
- }
- // clear moved data in left range
- helper.clear(id, boundary);
- return targetKVSpaceWriter;
- } catch (Throwable e) {
- throw new KVEngineException("Delete range in batch failed", e);
- }
- }
-
- @Override
- public IKVSpaceMetadataWriter migrateFrom(String fromSpaceId, Boundary boundary) {
-
- try {
- InMemKVSpace<?, ?> sourceKVSpace = engine.createIfMissing(fromSpaceId);
- IKVSpaceWriter sourceKVSpaceWriter = sourceKVSpace.toWriter();
- // move data
- try (IKVSpaceIterator itr = sourceKVSpace.newIterator(boundary)) {
- for (itr.seekToFirst(); itr.isValid(); itr.next()) {
- helper.put(id, itr.key(), itr.value());
- }
- }
- // clear moved data in right range
- sourceKVSpaceWriter.clear(boundary);
- return sourceKVSpaceWriter;
- } catch (Throwable e) {
- throw new KVEngineException("Delete range in batch failed", e);
- }
- }
-
@Override
public void done() {
opMeters.batchWriteCallTimer.record(() -> {
@@ -167,14 +133,4 @@ class InMemKVSpaceWriter<E extends InMemKVEngine<E, T>, T extends InMemKVSpace<E
public int count() {
return helper.count();
}
-
- @Override
- protected Map<ByteString, ByteString> metadataMap() {
- return metadataMap;
- }
-
- @Override
- protected ConcurrentSkipListMap<ByteString, ByteString> rangeData() {
- return rangeData;
- }
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriterHelper.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriterHelper.java
similarity index 57%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriterHelper.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriterHelper.java
index 961f05036..b7edd39f9 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriterHelper.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemKVSpaceWriterHelper.java
@@ -19,8 +19,6 @@
package org.apache.bifromq.basekv.localengine.memory;
-import org.apache.bifromq.basekv.localengine.ISyncContext;
-import org.apache.bifromq.basekv.proto.Boundary;
import com.google.protobuf.ByteString;
import java.util.ArrayList;
import java.util.HashMap;
@@ -29,30 +27,28 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.proto.Boundary;
class InMemKVSpaceWriterHelper {
- private final Map<String, Map<ByteString, ByteString>> metadataMap;
- private final Map<String, ConcurrentSkipListMap<ByteString, ByteString>> rangeDataMap;
+ private final Map<String, InMemKVSpaceEpoch> kvSpaceEpochMap;
private final Map<String, WriteBatch> batchMap;
private final Map<String, Consumer<Boolean>> afterWriteCallbacks = new HashMap<>();
+ private final Map<String, Consumer<WriteImpact>> afterImpactCallbacks = new HashMap<>();
private final Map<String, Boolean> metadataChanges = new HashMap<>();
private final Set<ISyncContext.IMutator> mutators = new HashSet<>();
InMemKVSpaceWriterHelper() {
- this.metadataMap = new HashMap<>();
- this.rangeDataMap = new HashMap<>();
+ this.kvSpaceEpochMap = new HashMap<>();
this.batchMap = new HashMap<>();
}
- void addMutators(String rangeId,
- Map<ByteString, ByteString> metadata,
- ConcurrentSkipListMap<ByteString, ByteString> rangeData,
+ void addMutators(String id,
+ InMemKVSpaceEpoch epoch,
ISyncContext.IMutator mutator) {
- metadataMap.put(rangeId, metadata);
- rangeDataMap.put(rangeId, rangeData);
+ kvSpaceEpochMap.put(id, epoch);
mutators.add(mutator);
}
@@ -61,6 +57,9 @@ class InMemKVSpaceWriterHelper {
metadataChanges.put(rangeId, false);
}
+ void addAfterImpactCallback(String rangeId, Consumer<WriteImpact> afterImpact) {
+ afterImpactCallbacks.put(rangeId, afterImpact);
+ }
void metadata(String rangeId, ByteString metaKey, ByteString metaValue) {
batchMap.computeIfAbsent(rangeId, k -> new WriteBatch(rangeId)).metadata(metaKey, metaValue);
@@ -84,24 +83,39 @@ class InMemKVSpaceWriterHelper {
}
void done() {
- Runnable doneFn = () -> batchMap.values().forEach(WriteBatch::end);
- AtomicReference<Runnable> finalRun = new AtomicReference<>();
+ ISyncContext.IMutation doneFn = () -> {
+ Map<String, WriteImpact> impacts = new HashMap<>();
+ batchMap.values().forEach(batch -> {
+ WriteImpact impact = batch.endAndCollectImpact();
+ if (impact != null) {
+ impacts.put(batch.rangeId, impact);
+ }
+ });
+ impacts.forEach((id, imp) -> {
+ Consumer<WriteImpact> cb = afterImpactCallbacks.get(id);
+ if (cb != null) {
+ cb.accept(imp);
+ }
+ });
+ return false;
+ };
+ AtomicReference<ISyncContext.IMutation> finalRun = new AtomicReference<>();
for (ISyncContext.IMutator mutator : mutators) {
if (finalRun.get() == null) {
finalRun.set(() -> mutator.run(doneFn));
} else {
- Runnable innerRun = finalRun.get();
+ ISyncContext.IMutation innerRun = finalRun.get();
finalRun.set(() -> mutator.run(innerRun));
}
}
- finalRun.get().run();
+ finalRun.get().mutate();
for (String rangeId : afterWriteCallbacks.keySet()) {
afterWriteCallbacks.get(rangeId).accept(metadataChanges.get(rangeId));
}
}
void abort() {
- rangeDataMap.clear();
+ batchMap.clear();
}
int count() {
@@ -111,7 +125,12 @@ class InMemKVSpaceWriterHelper {
protected interface KVAction {
KVAction.Type type();
- enum Type {Put, Delete, DeleteRange}
+ enum Type { Put, Delete, DeleteRange }
+ }
+
+ public record WriteImpact(List<ByteString> pointKeys,
+ List<Boundary> deleteRanges,
+ Map<ByteString, Integer> pointDeltaBytes) {
}
protected class WriteBatch {
@@ -119,8 +138,8 @@ class InMemKVSpaceWriterHelper {
Map<ByteString, ByteString> metadata = new HashMap<>();
List<KVAction> actions = new ArrayList<>();
- protected WriteBatch(String rangeId) {
- this.rangeId = rangeId;
+ protected WriteBatch(String spaceId) {
+ this.rangeId = spaceId;
}
public void metadata(ByteString key, ByteString value) {
@@ -147,43 +166,78 @@ class InMemKVSpaceWriterHelper {
actions.add(new WriteBatch.DeleteRange(boundary));
}
- public void end() {
- metadataMap.get(rangeId).putAll(metadata);
+ public WriteImpact endAndCollectImpact() {
+ List<ByteString> putOrDeleteKeys = new ArrayList<>();
+ List<Boundary> deleteRanges = new ArrayList<>();
+ Map<ByteString, Integer> pointDeltaBytes = new HashMap<>();
+ InMemKVSpaceEpoch epoch = kvSpaceEpochMap.get(rangeId);
+
+ metadata.forEach(epoch::setMetadata);
+
for (KVAction action : actions) {
switch (action.type()) {
case Put -> {
WriteBatch.Put put = (WriteBatch.Put) action;
- rangeDataMap.get(rangeId).put(put.key, put.value);
+ // compute delta before mutation
+ ByteString old = epoch.dataMap().get(put.key);
+ int oldBytes = old == null ? 0 : (put.key.size() + old.size());
+ int newBytes = put.key.size() + put.value.size();
+ int delta = newBytes - oldBytes;
+ if (delta != 0) {
+ pointDeltaBytes.merge(put.key, delta, Integer::sum);
+ }
+ epoch.putData(put.key, put.value);
+ putOrDeleteKeys.add(put.key);
}
case Delete -> {
WriteBatch.Delete delete = (WriteBatch.Delete) action;
- rangeDataMap.get(rangeId).remove(delete.key);
+ // compute delta before mutation
+ ByteString old = epoch.dataMap().get(delete.key);
+ if (old != null) {
+ int delta = -(delete.key.size() + old.size());
+ pointDeltaBytes.merge(delete.key, delta, Integer::sum);
+ epoch.removeData(delete.key);
+ }
+ putOrDeleteKeys.add(delete.key);
}
case DeleteRange -> {
WriteBatch.DeleteRange deleteRange = (WriteBatch.DeleteRange) action;
Boundary boundary = deleteRange.boundary;
NavigableSet<ByteString> inRangeKeys;
if (!boundary.hasStartKey() && !boundary.hasEndKey()) {
- inRangeKeys = rangeDataMap.get(rangeId).keySet();
+ inRangeKeys = epoch.dataMap().navigableKeySet();
} else if (!boundary.hasStartKey()) {
- inRangeKeys = rangeDataMap.get(rangeId).headMap(boundary.getEndKey()).keySet();
+ inRangeKeys = epoch.dataMap().headMap(boundary.getEndKey(), false).navigableKeySet();
} else if (!boundary.hasEndKey()) {
- inRangeKeys = rangeDataMap.get(rangeId).tailMap(boundary.getStartKey()).keySet();
+ inRangeKeys = epoch.dataMap().tailMap(boundary.getStartKey(), true).navigableKeySet();
} else {
- inRangeKeys =
- rangeDataMap.get(rangeId).subMap(boundary.getStartKey(), boundary.getEndKey()).keySet();
+ inRangeKeys = epoch.dataMap()
+ .subMap(boundary.getStartKey(), true, boundary.getEndKey(), false)
+ .navigableKeySet();
+ }
+ // accumulate negative delta per key before removal
+ for (ByteString k : inRangeKeys) {
+ ByteString v = epoch.dataMap().get(k);
+ if (v != null) {
+ int delta = -(k.size() + v.size());
+ pointDeltaBytes.merge(k, delta, Integer::sum);
+ }
}
- inRangeKeys.forEach(k -> rangeDataMap.get(rangeId).remove(k));
+ inRangeKeys.forEach(epoch::removeData);
+ deleteRanges.add(boundary);
+ }
+ default -> {
+ // no-op
}
}
}
+ return new WriteImpact(putOrDeleteKeys, deleteRanges, pointDeltaBytes);
}
public void abort() {
}
-
record Put(ByteString key, ByteString value) implements KVAction {
@Override
public Type type() {
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVEngine.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVEngine.java
similarity index 73%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVEngine.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVEngine.java
index 838055af0..a35d6d73b 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVEngine.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVEngine.java
@@ -19,21 +19,28 @@
package org.apache.bifromq.basekv.localengine.memory;
+import com.google.protobuf.Struct;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.slf4j.Logger;
-public class InMemWALableKVEngine extends InMemKVEngine<InMemWALableKVEngine, InMemWALableKVSpace> {
- public InMemWALableKVEngine(String overrideIdentity, InMemKVEngineConfigurator c) {
- super(overrideIdentity, c);
+class InMemWALableKVEngine extends InMemKVEngine<InMemWALableKVEngine, InMemWALableKVSpace> {
+
+ InMemWALableKVEngine(String overrideIdentity, Struct conf) {
+ super(overrideIdentity, conf);
}
@Override
protected InMemWALableKVSpace doBuildKVSpace(String spaceId,
- InMemKVEngineConfigurator configurator,
+ Struct conf,
Runnable onDestroy,
KVSpaceOpMeters opMeters,
Logger logger,
String... tags) {
- return new InMemWALableKVSpace(spaceId, configurator, this, onDestroy, opMeters, logger);
+ return new InMemWALableKVSpace(spaceId, conf, this, onDestroy, opMeters, logger);
+ }
+
+ @Override
+ protected Struct defaultConf() {
+ return InMemDefaultConfigs.WAL;
}
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
similarity index 51%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
rename to base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
index 38b2b53c8..3824aa97f 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
+++ b/base-kv/base-kv-local-engine-memory/src/main/java/org/apache/bifromq/basekv/localengine/memory/InMemWALableKVSpace.java
@@ -19,24 +19,56 @@
package org.apache.bifromq.basekv.localengine.memory;
+import com.google.protobuf.Struct;
+import java.util.concurrent.CompletableFuture;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
-import java.util.concurrent.CompletableFuture;
import org.slf4j.Logger;
-public class InMemWALableKVSpace extends InMemKVSpace<InMemWALableKVEngine, InMemWALableKVSpace>
+class InMemWALableKVSpace extends InMemKVSpace<InMemWALableKVEngine, InMemWALableKVSpace>
implements IWALableKVSpace {
- protected InMemWALableKVSpace(String id,
- InMemKVEngineConfigurator configurator,
- InMemWALableKVEngine engine,
- Runnable onDestroy,
- KVSpaceOpMeters opMeters,
- Logger logger) {
- super(id, configurator, engine, onDestroy, opMeters, logger);
+ private final InMemKVSpaceEpoch epoch;
+
+ InMemWALableKVSpace(String id,
+ Struct conf,
+ InMemWALableKVEngine engine,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ super(id, conf, engine, onDestroy, opMeters, logger, tags);
+ epoch = new InMemKVSpaceEpoch();
}
@Override
public CompletableFuture<Long> flush() {
return CompletableFuture.completedFuture(System.nanoTime());
}
+
+ @Override
+ public IKVSpaceWriter toWriter() {
+ return new InMemKVSpaceWriter<>(id, epoch, engine, syncContext, metadataUpdated -> {
+ if (metadataUpdated) {
+ this.loadMetadata();
+ }
+ }, impact -> {
+ tracker.updateOnWrite(impact, epoch.dataMap());
+ }, opMeters, logger);
+ }
+
+ @Override
+ protected void doClose() {
+
+ }
+
+ @Override
+ protected InMemKVSpaceEpoch handle() {
+ return epoch;
+ }
+
+ @Override
+ protected void doOpen() {
+
+ }
}
diff --git a/base-kv/base-kv-local-engine-memory/src/main/resources/META-INF/services/org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider b/base-kv/base-kv-local-engine-memory/src/main/resources/META-INF/services/org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider
new file mode 100644
index 000000000..2cc3573a2
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/main/resources/META-INF/services/org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider
@@ -0,0 +1 @@
+org.apache.bifromq.basekv.localengine.memory.InMemKVEngineProvider
diff --git a/base-kv/base-kv-local-engine-memory/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVSpaceRestoreFlushListenerTest.java b/base-kv/base-kv-local-engine-memory/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVSpaceRestoreFlushListenerTest.java
new file mode 100644
index 000000000..a7f46bc21
--- /dev/null
+++ b/base-kv/base-kv-local-engine-memory/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemCPableKVSpaceRestoreFlushListenerTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.memory;
+
+import static org.testng.Assert.assertEquals;
+
+import com.google.protobuf.ByteString;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class InMemCPableKVSpaceRestoreFlushListenerTest {
+ private IKVEngine<? extends ICPableKVSpace> engine;
+
+ @BeforeMethod
+ public void setup() {
+ engine = new InMemCPableKVEngine(null, InMemDefaultConfigs.CP);
+ engine.start();
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ if (engine != null) {
+ engine.stop();
+ }
+ }
+
+ @Test
+ public void testDoneReportsSingleAggregatedFlush() {
+ String spaceId = "inmem_flush";
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+
+ AtomicInteger callbackCount = new AtomicInteger();
+ AtomicLong totalEntries = new AtomicLong();
+ AtomicLong totalBytes = new AtomicLong();
+ IRestoreSession session = space.startRestore((c, b) -> {
+ callbackCount.incrementAndGet();
+ totalEntries.addAndGet(c);
+ totalBytes.addAndGet(b);
+ });
+
+ int n = 1000;
+ long expectBytes = 0;
+ for (int i = 0; i < n; i++) {
+ ByteString k = ByteString.copyFromUtf8("k" + i);
+ ByteString v = ByteString.copyFromUtf8("v" + i);
+ expectBytes += k.size() + v.size();
+ session.put(k, v);
+ }
+ session.done();
+
+ assertEquals(callbackCount.get(), 1);
+ assertEquals(totalEntries.get(), n);
+ assertEquals(totalBytes.get(), expectBytes);
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java b/base-kv/base-kv-local-engine-memory/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
similarity index 73%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
rename to base-kv/base-kv-local-engine-memory/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
index 28cb205bc..9522051e8 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-memory/src/test/java/org/apache/bifromq/basekv/localengine/memory/InMemKVEngineTest.java
@@ -22,10 +22,17 @@ package org.apache.bifromq.basekv.localengine.memory;
import org.apache.bifromq.basekv.localengine.AbstractKVEngineTest;
import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
-public class InMemKVEngineTest extends AbstractKVEngineTest {
+public class InMemKVEngineTest extends AbstractKVEngineTest<ICPableKVSpace> {
@Override
protected IKVEngine<? extends ICPableKVSpace> newEngine() {
- return new InMemCPableKVEngine(null, new InMemKVEngineConfigurator());
+ return new InMemCPableKVEngine(null, InMemDefaultConfigs.CP);
+ }
+
+ @Override
+ protected IKVSpaceWriter writerOf(ICPableKVSpace space) {
+ // InMem CPable space returns a restorable writer which is also an IKVSpaceWriter
+ return space.toWriter();
}
}
diff --git a/base-kv/base-kv-local-engine/src/test/resources/log4j2-test.xml b/base-kv/base-kv-local-engine-memory/src/test/resources/log4j2-test.xml
similarity index 99%
copy from base-kv/base-kv-local-engine/src/test/resources/log4j2-test.xml
copy to base-kv/base-kv-local-engine-memory/src/test/resources/log4j2-test.xml
index 7b038d5b4..1404c8668 100644
--- a/base-kv/base-kv-local-engine/src/test/resources/log4j2-test.xml
+++ b/base-kv/base-kv-local-engine-memory/src/test/resources/log4j2-test.xml
@@ -35,3 +35,4 @@
</Root>
</Loggers>
</Configuration>
+
diff --git a/base-kv/base-kv-local-engine/pom.xml b/base-kv/base-kv-local-engine-rocksdb/pom.xml
similarity index 67%
copy from base-kv/base-kv-local-engine/pom.xml
copy to base-kv/base-kv-local-engine-rocksdb/pom.xml
index 398e6036a..81ea28921 100644
--- a/base-kv/base-kv-local-engine/pom.xml
+++ b/base-kv/base-kv-local-engine-rocksdb/pom.xml
@@ -17,7 +17,6 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -27,54 +26,29 @@
<artifactId>base-kv</artifactId>
<version>${revision}</version>
</parent>
-
- <artifactId>base-kv-local-engine</artifactId>
+ <artifactId>base-kv-local-engine-rocksdb</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.bifromq</groupId>
- <artifactId>base-kv-type-proto</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.bifromq</groupId>
- <artifactId>base-env-provider</artifactId>
+ <artifactId>base-kv-local-engine-spi</artifactId>
</dependency>
<dependency>
<groupId>org.apache.bifromq</groupId>
- <artifactId>base-logger</artifactId>
- </dependency>
- <dependency>
- <groupId>io.reactivex.rxjava3</groupId>
- <artifactId>rxjava</artifactId>
- </dependency>
- <dependency>
- <groupId>io.micrometer</groupId>
- <artifactId>micrometer-core</artifactId>
+ <artifactId>base-kv-local-engine-spi</artifactId>
+ <type>test-jar</type>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <!-- for testing-->
+ <!-- test -->
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -108,12 +82,4 @@
<scope>test</scope>
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
+</project>
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVSpaceReader.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVSpaceReader.java
new file mode 100644
index 000000000..c510f5f86
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVSpaceReader.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static com.google.protobuf.UnsafeByteOperations.unsafeWrap;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.toDataKey;
+
+import com.google.protobuf.ByteString;
+import java.util.Optional;
+import org.apache.bifromq.basekv.localengine.AbstractKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksDBException;
+import org.slf4j.Logger;
+
+abstract class AbstractRocksDBKVSpaceReader extends AbstractKVSpaceReader {
+ protected AbstractRocksDBKVSpaceReader(String id,
+ KVSpaceOpMeters opMeters,
+ Logger logger) {
+ super(id, opMeters, logger);
+ }
+
+ protected abstract IRocksDBKVSpaceEpoch handle();
+
+ protected abstract RocksDBSnapshot snapshot();
+
+ protected final long doSize(Boundary boundary) {
+ return RocksDBHelper.sizeOfBoundary(handle(), boundary);
+ }
+
+ @Override
+ protected final boolean doExist(ByteString key) {
+ return get(key).isPresent();
+ }
+
+ @Override
+ protected final Optional<ByteString> doGet(ByteString key) {
+ try (ReadOptions readOptions = new ReadOptions()) {
+ readOptions.setSnapshot(snapshot().snapshot());
+ IRocksDBKVSpaceEpoch dbHandle = handle();
+ byte[] data = dbHandle.db().get(dbHandle.cf(), readOptions, toDataKey(key));
+ return Optional.ofNullable(data == null ? null : unsafeWrap(data));
+ } catch (RocksDBException rocksDBException) {
+ throw new KVEngineException("Get failed", rocksDBException);
+ }
+ }
+}
diff --git a/base-kv/base-kv-store-server/src/main/java/org/apache/bifromq/basekv/store/range/AdaptiveWriteBudget.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AdaptiveWriteBudget.java
similarity index 95%
rename from base-kv/base-kv-store-server/src/main/java/org/apache/bifromq/basekv/store/range/AdaptiveWriteBudget.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AdaptiveWriteBudget.java
index 6c95a30fe..a5a7622eb 100644
--- a/base-kv/base-kv-store-server/src/main/java/org/apache/bifromq/basekv/store/range/AdaptiveWriteBudget.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AdaptiveWriteBudget.java
@@ -17,8 +17,10 @@
* under the License.
*/
-package org.apache.bifromq.basekv.store.range;
+package org.apache.bifromq.basekv.localengine.rocksdb;
+// Package-visible adaptive write budget copied from base-kv-store-server module
+// to avoid cross-package visibility; used by RestoreSession to tune flush cadence.
final class AdaptiveWriteBudget {
private static final long TARGET_LATENCY_MS = 50;
private static final double EMA_ALPHA = 0.3;
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AutoCleaner.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AutoCleaner.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AutoCleaner.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/AutoCleaner.java
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceCheckpoint.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceCheckpoint.java
similarity index 100%
copy from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceCheckpoint.java
copy to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceCheckpoint.java
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceEpoch.java
similarity index 63%
copy from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java
copy to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceEpoch.java
index a780d4321..64822a540 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceEpoch.java
@@ -19,20 +19,25 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-interface IWriteStatsRecorder {
- IRecorder newRecorder();
+import org.apache.bifromq.basekv.localengine.IKVSpaceEpoch;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.RocksDB;
- void reset();
-
- interface IRecorder {
- void recordPut();
-
- void recordInsert();
-
- void recordDelete();
-
- void recordDeleteRange();
-
- void stop();
- }
+/**
+ * Interface for accessing RocksDB space epoch.
+ */
+interface IRocksDBKVSpaceEpoch extends IKVSpaceEpoch {
+ /**
+ * Get the backing RocksDB instance.
+ *
+ * @return the backing RocksDB instance
+ */
+ RocksDB db();
+
+ /**
+ * Get the column family handle.
+ *
+ * @return the column family handle
+ */
+ ColumnFamilyHandle cf();
}
diff --git a/base-kv/base-kv-store-server/src/main/java/org/apache/bifromq/basekv/store/range/IKVCheckpointIterator.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceEpochHandle.java
similarity index 77%
rename from base-kv/base-kv-store-server/src/main/java/org/apache/bifromq/basekv/store/range/IKVCheckpointIterator.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceEpochHandle.java
index 42199c551..d78f9bc36 100644
--- a/base-kv/base-kv-store-server/src/main/java/org/apache/bifromq/basekv/store/range/IKVCheckpointIterator.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IRocksDBKVSpaceEpochHandle.java
@@ -17,11 +17,14 @@
* under the License.
*/
-package org.apache.bifromq.basekv.store.range;
+package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.store.api.IKVIterator;
-
-public interface IKVCheckpointIterator extends IKVIterator, AutoCloseable {
- @Override
+/**
+ * Handle to a RocksDB KV space epoch.
+ */
+interface IRocksDBKVSpaceEpochHandle extends IRocksDBKVSpaceEpoch {
+ /**
+ * Close the handle and release any resources.
+ */
void close();
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IWriteStatsRecorder.java
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/IKVEngineConfigurator.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IteratorOptions.java
similarity index 84%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/IKVEngineConfigurator.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IteratorOptions.java
index a2ed7ab70..b7e5d6dad 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/IKVEngineConfigurator.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/IteratorOptions.java
@@ -14,10 +14,10 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
-package org.apache.bifromq.basekv.localengine;
+package org.apache.bifromq.basekv.localengine.rocksdb;
-public interface IKVEngineConfigurator {
+record IteratorOptions(boolean fillCache, long readAheadSize) {
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/Keys.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/Keys.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/Keys.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/Keys.java
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/NoopWriteStatsRecorder.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/NoopWriteStatsRecorder.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/NoopWriteStatsRecorder.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/NoopWriteStatsRecorder.java
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngine.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngine.java
similarity index 71%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngine.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngine.java
index 1f0b8e1ea..681203185 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngine.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngine.java
@@ -19,24 +19,28 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.localengine.KVEngineException;
-import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import static org.apache.bifromq.basekv.localengine.StructUtil.strVal;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
+
+import com.google.protobuf.Struct;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import java.io.File;
import java.nio.file.Files;
+import java.nio.file.Paths;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.slf4j.Logger;
-public class RocksDBCPableKVEngine extends
- RocksDBKVEngine<RocksDBCPableKVEngine, RocksDBCPableKVSpace, RocksDBCPableKVEngineConfigurator> {
+class RocksDBCPableKVEngine extends RocksDBKVEngine<RocksDBCPableKVSpace> {
private final File cpRootDir;
private MetricManager metricManager;
- public RocksDBCPableKVEngine(String overrideIdentity,
- RocksDBCPableKVEngineConfigurator configurator) {
- super(overrideIdentity, configurator);
- cpRootDir = new File(configurator.dbCheckpointRootDir());
+ public RocksDBCPableKVEngine(String overrideIdentity, Struct conf) {
+ super(overrideIdentity, conf);
+ cpRootDir = new File(strVal(conf, DB_CHECKPOINT_ROOT_DIR));
try {
Files.createDirectories(cpRootDir.getAbsoluteFile().toPath());
} catch (Throwable e) {
@@ -46,12 +50,27 @@ public class RocksDBCPableKVEngine extends
@Override
protected RocksDBCPableKVSpace doBuildKVSpace(String spaceId,
- RocksDBCPableKVEngineConfigurator configurator,
+ Struct conf,
Runnable onDestroy,
KVSpaceOpMeters opMeters,
Logger logger,
String... tags) {
- return new RocksDBCPableKVSpace(spaceId, configurator, this, onDestroy, opMeters, logger, tags);
+ return new RocksDBCPableKVSpace(spaceId, conf, this, onDestroy, opMeters, logger, tags);
+ }
+
+ @Override
+ protected Struct defaultConf() {
+ return RocksDBDefaultConfigs.CP;
+ }
+
+ @Override
+ protected void validateSemantics(Struct conf) {
+ try {
+ Paths.get(strVal(conf, DB_ROOT_DIR));
+ Paths.get(strVal(conf, DB_CHECKPOINT_ROOT_DIR));
+ } catch (Throwable t) {
+ throw new IllegalArgumentException("Invalid RocksDB data/checkpoint path in configuration", t);
+ }
}
@Override
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpace.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpace.java
new file mode 100644
index 000000000..665fe8f52
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpace.java
@@ -0,0 +1,556 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.bifromq.basekv.localengine.StructUtil.strVal;
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getGauge;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.LATEST_CP_KEY;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.deleteDir;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.Timer;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceCheckpoint;
+import org.apache.bifromq.basekv.localengine.IKVSpaceMigratableWriter;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.RestoreMode;
+import org.apache.bifromq.basekv.localengine.metrics.GeneralKVSpaceMetric;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
+import org.rocksdb.FlushOptions;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+
+class RocksDBCPableKVSpace extends RocksDBKVSpace implements ICPableKVSpace {
+ public static final String ACTIVE_GEN_POINTER = "ACTIVE";
+ private static final String CP_SUFFIX = ".cp";
+ private final RocksDBCPableKVEngine engine;
+ private final File cpRootDir;
+ private final WriteOptions writeOptions;
+ private final AtomicReference<String> latestCheckpointId = new AtomicReference<>();
+ private final Cache<String, IRocksDBKVSpaceCheckpoint> checkpoints;
+ private final MetricManager metricMgr;
+ private final AtomicReference<RocksDBCPableKVSpaceEpochHandle> active = new AtomicReference<>();
+ private File currentDBDir;
+ // keep a strong ref to latest checkpoint
+ private IKVSpaceCheckpoint latestCheckpoint;
+
+ @SneakyThrows
+ RocksDBCPableKVSpace(String id,
+ Struct conf,
+ RocksDBCPableKVEngine engine,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ super(id, conf, engine, onDestroy, opMeters, logger, tags);
+ this.engine = engine;
+ cpRootDir = new File(strVal(conf, DB_CHECKPOINT_ROOT_DIR), id);
+ checkpoints = Caffeine.newBuilder().weakValues().build();
+ writeOptions = new WriteOptions().setDisableWAL(true);
+ Files.createDirectories(cpRootDir.getAbsoluteFile().toPath());
+ metricMgr = new MetricManager(tags);
+ // Ensure space root exists and initialize current generation directory.
+ // The legacy layout migration is done at constructor time to keep open() path clean.
+ Files.createDirectories(spaceRootDir().getAbsoluteFile().toPath());
+ initOrMigrateCurrentDBDir();
+ }
+
+ @Override
+ protected WriteOptions writeOptions() {
+ return writeOptions;
+ }
+
+ @Override
+ protected RocksDBCPableKVSpaceEpochHandle handle() {
+ return active.get();
+ }
+
+ @Override
+ public String checkpoint() {
+ return metricMgr.checkpointTimer.record(() -> {
+ synchronized (this) {
+ IRocksDBKVSpaceCheckpoint cp = doCheckpoint();
+ checkpoints.put(cp.cpId(), cp);
+ latestCheckpoint = cp;
+ return cp.cpId();
+ }
+ });
+ }
+
+ @Override
+ public Optional<IKVSpaceCheckpoint> openCheckpoint(String checkpointId) {
+ IRocksDBKVSpaceCheckpoint cp = checkpoints.getIfPresent(checkpointId);
+ return Optional.ofNullable(cp);
+ }
+
+ @Override
+ public IRestoreSession startRestore(IRestoreSession.FlushListener flushListener) {
+ return new RestoreSession(RestoreMode.Replace, flushListener, logger);
+ }
+
+ @Override
+ public IRestoreSession startReceiving(IRestoreSession.FlushListener flushListener) {
+ return new RestoreSession(RestoreMode.Overlay, flushListener, logger);
+ }
+
+ @Override
+ public IKVSpaceMigratableWriter toWriter() {
+ return new RocksDBKVSpaceMigratableWriter(id, active.get(), engine, writeOptions(), syncContext,
+ writeStats.newRecorder(), this::publishMetadata, opMeters, logger);
+ }
+
+ @Override
+ protected void doClose() {
+ logger.debug("Flush RocksDBCPableKVSpace[{}] before closing", id);
+ try (FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) {
+ active.get().db.flush(flushOptions);
+ } catch (Throwable e) {
+ logger.error("Flush RocksDBCPableKVSpace[{}] error", id, e);
+ }
+ metricMgr.close();
+ checkpoints.asMap().forEach((cpId, cp) -> cp.close());
+ writeOptions.close();
+ RocksDBCPableKVSpaceEpochHandle h = active.get();
+ if (h != null) {
+ h.close();
+ }
+ super.doClose();
+ }
+
+ @Override
+ protected void doDestroy() {
+ try {
+ deleteDir(cpRootDir.toPath());
+ } catch (IOException e) {
+ logger.error("Failed to delete checkpoint root dir: {}", cpRootDir, e);
+ } finally {
+ super.doDestroy();
+ }
+ }
+
+ @Override
+ protected void doOpen() {
+ try {
+ // Use currentDBDir initialized during construction
+ this.active.set(newEpochHandle(currentDBDir));
+ // cleanup inactive generations at startup
+ cleanInactiveOnStartup();
+ loadLatestCheckpoint();
+ super.doOpen();
+ } catch (Throwable e) {
+ throw new KVEngineException("Failed to open CPable KVSpace", e);
+ }
+ }
+
+ private RocksDBCPableKVSpaceEpochHandle newEpochHandle(File dir) {
+ return new RocksDBCPableKVSpaceEpochHandle(id, dir, this.conf, this::isRetired, logger, tags);
+ }
+
+ private IRocksDBKVSpaceCheckpoint doCheckpoint() {
+ String cpId = genCheckpointId();
+ File cpDir = Paths.get(cpRootDir.getAbsolutePath(), cpId).toFile();
+ try {
+ logger.debug("KVSpace[{}] checkpoint start: checkpointId={}", id, cpId);
+ RocksDBCPableKVSpaceEpochHandle currentHandle = active.get();
+ currentHandle.db.put(currentHandle.cf, LATEST_CP_KEY, cpId.getBytes());
+ currentHandle.checkpoint.createCheckpoint(cpDir.toString());
+ latestCheckpointId.set(cpId);
+ return new RocksDBKVSpaceCheckpoint(id, cpId, cpDir, this::isLatest, opMeters, logger);
+ } catch (Throwable e) {
+ throw new KVEngineException("Checkpoint key range error", e);
+ }
+ }
+
+ @SneakyThrows
+ private IRocksDBKVSpaceCheckpoint doLoadLatestCheckpoint() {
+ RocksDBCPableKVSpaceEpochHandle currentHandle = active.get();
+ byte[] cpIdBytes = currentHandle.db.get(currentHandle.cf, LATEST_CP_KEY);
+ if (cpIdBytes != null) {
+ try {
+ String cpId = new String(cpIdBytes, UTF_8);
+ File cpDir = Paths.get(cpRootDir.getAbsolutePath(), cpId).toFile();
+ // cleanup obsolete checkpoints
+ for (String obsoleteId : obsoleteCheckpoints(cpId)) {
+ try {
+ cleanCheckpoint(obsoleteId);
+ } catch (Throwable e) {
+ logger.error("Clean checkpoint[{}] for kvspace[{}] error", obsoleteId, id, e);
+ }
+ }
+ logger.debug("Load latest checkpoint[{}] of kvspace[{}] in engine[{}] at path[{}]",
+ cpId, id, engine.id(), cpDir);
+ latestCheckpointId.set(cpId);
+ return new RocksDBKVSpaceCheckpoint(id, cpId, cpDir, this::isLatest, opMeters, logger);
+ } catch (Throwable e) {
+ logger.warn("Failed to load latest checkpoint, checkpoint now", e);
+ }
+ }
+ return doCheckpoint();
+ }
+
+ @SneakyThrows
+ private void loadLatestCheckpoint() {
+ IRocksDBKVSpaceCheckpoint checkpoint = doLoadLatestCheckpoint();
+ assert !checkpoints.asMap().containsKey(checkpoint.cpId());
+ checkpoints.put(checkpoint.cpId(), checkpoint);
+ latestCheckpoint = checkpoint;
+ }
+
+ private String genCheckpointId() {
+ // we need generate global unique checkpoint id, since it will be used in raft snapshot
+ return UUID.randomUUID() + CP_SUFFIX;
+ }
+
+ private boolean isLatest(String cpId) {
+ return cpId.equals(latestCheckpointId.get());
+ }
+
+ private File checkpointDir(String cpId) {
+ return Paths.get(cpRootDir.getAbsolutePath(), cpId).toFile();
+ }
+
+ private boolean isRetired(String genId) {
+ // A generation is retired if it is not the one pointed by ACTIVE_GEN_POINTER
+ File pointer = new File(spaceRootDir(), ACTIVE_GEN_POINTER);
+ try {
+ if (!pointer.exists()) {
+ // No pointer means no active generation defined; treat all as retired.
+ return true;
+ }
+ String activeUuid = Files.readString(pointer.toPath()).trim();
+ return !genId.equals(activeUuid);
+ } catch (Throwable ignore) {
+ // On any error, be conservative and treat as retired to avoid leak
+ return true;
+ }
+ }
+
+ private Iterable<String> obsoleteCheckpoints(String skipId) {
+ File[] cpDirList = cpRootDir.listFiles();
+ if (cpDirList == null) {
+ return Collections.emptyList();
+ }
+ return Arrays.stream(cpDirList)
+ .filter(File::isDirectory)
+ .map(File::getName)
+ .filter(cpId -> !skipId.equals(cpId))
+ .collect(Collectors.toList());
+ }
+
+ private void cleanCheckpoint(String cpId) {
+ logger.debug("Delete checkpoint[{}] of kvspace[{}]", cpId, id);
+ try {
+ deleteDir(checkpointDir(cpId).toPath());
+ } catch (IOException e) {
+ logger.error("Failed to clean checkpoint[{}] for kvspace[{}] at path:{}", cpId, id, checkpointDir(cpId));
+ }
+ }
+
+ private void switchTo(RocksDBCPableKVSpaceEpochHandle handle) {
+ syncContext.mutator().run(() -> {
+ // inactive handle will be closed and cleaned up by cleaner
+ active.set(handle);
+ updateCurrentDBDir(handle.dir);
+ reloadMetadata();
+ return true;
+ });
+ }
+
+ @SneakyThrows
+ private void initOrMigrateCurrentDBDir() {
+ File spaceRoot = spaceRootDir();
+ File pointer = new File(spaceRoot, ACTIVE_GEN_POINTER);
+ // If pointer exists, honor it; otherwise detect legacy layout and migrate.
+ if (pointer.exists()) {
+ try {
+ String uuid = Files.readString(pointer.toPath()).trim();
+ if (!uuid.isEmpty()) {
+ File dir = new File(spaceRoot, uuid);
+ if (dir.exists() && dir.isDirectory()) {
+ currentDBDir = dir;
+ return;
+ }
+ }
+ } catch (Throwable t) {
+ // fall through to create a new generation
+ logger.warn("Failed to read {} for {}, create new generation", ACTIVE_GEN_POINTER, id, t);
+ }
+ // pointer invalid or target missing -> create a new generation
+ File newGen = new File(spaceRoot, UUID.randomUUID().toString());
+ Files.createDirectories(newGen.toPath());
+ updateCurrentDBDir(newGen);
+ return;
+ }
+
+ // No pointer file, check if space root is empty: empty means fresh deploy; non-empty means legacy layout.
+ String[] children = spaceRoot.list();
+ if (children == null || children.length == 0) {
+ // Fresh deployment, bootstrap new generation
+ File newGen = new File(spaceRoot, UUID.randomUUID().toString());
+ Files.createDirectories(newGen.toPath());
+ updateCurrentDBDir(newGen);
+ return;
+ }
+
+ // Legacy layout: move all existing contents under a new generation directory, then write pointer file.
+ File newGen = new File(spaceRoot, UUID.randomUUID().toString());
+ Files.createDirectories(newGen.toPath());
+ File[] entries = spaceRoot.listFiles();
+ if (entries != null) {
+ for (File entry : entries) {
+ if (entry.getName().equals(ACTIVE_GEN_POINTER)) {
+ continue;
+ }
+ try {
+ Path target = new File(newGen, entry.getName()).toPath();
+ Files.move(entry.toPath(), target);
+ } catch (Throwable moveEx) {
+ logger.warn("Failed to move legacy entry {} for space[{}] by Files.move, fallback to rename",
+ entry.getAbsolutePath(), id, moveEx);
+ // Fallback to rename if Files.move fails for any reason
+ boolean renamed = entry.renameTo(new File(newGen, entry.getName()));
+ if (!renamed) {
+ logger.warn("Failed to move legacy entry {} for space[{}]", entry.getAbsolutePath(), id);
+ }
+ }
+ }
+ }
+ updateCurrentDBDir(newGen);
+ }
+
+ private void updateCurrentDBDir(File newDir) {
+ this.currentDBDir = newDir;
+ File pointer = new File(spaceRootDir(), ACTIVE_GEN_POINTER);
+ try {
+ Files.writeString(pointer.toPath(), newDir.getName());
+ } catch (Throwable t) {
+ logger.warn("Failed to update {} pointer for {}", ACTIVE_GEN_POINTER, id, t);
+ }
+ }
+
+ @SneakyThrows
+ private void cleanInactiveOnStartup() {
+ File pointer = new File(spaceRootDir(), ACTIVE_GEN_POINTER);
+ if (!pointer.exists()) {
+ return;
+ }
+ String activeUuid;
+ try {
+ activeUuid = Files.readString(pointer.toPath()).trim();
+ } catch (Throwable t) {
+ return;
+ }
+ if (activeUuid.isEmpty()) {
+ return;
+ }
+ File root = spaceRootDir();
+ File[] children = root.listFiles(File::isDirectory);
+ if (children != null) {
+ for (File c : children) {
+ if (!c.getName().equals(activeUuid)) {
+ deleteDir(c.toPath());
+ }
+ }
+ }
+ File[] files = root.listFiles(File::isFile);
+ if (files != null) {
+ for (File f : files) {
+ if (!f.getName().equals(ACTIVE_GEN_POINTER)) {
+ Files.deleteIfExists(f.toPath());
+ }
+ }
+ }
+ }
+
+ @Override
+ public IKVSpaceRefreshableReader reader() {
+ return new RocksDBKVSpaceReader(id, opMeters, logger, syncContext.refresher(), this::handle,
+ this::currentMetadata, new IteratorOptions(true, 0));
+ }
+
+ private class RestoreSession implements IRestoreSession {
+ private final File stagingDir;
+ private final RocksDBKVSpaceWriterHelper helper;
+ private final AdaptiveWriteBudget adaptiveWriteBudget = new AdaptiveWriteBudget();
+ private final IRestoreSession.FlushListener flushListener;
+ private final Logger logger;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+ private final RocksDBCPableKVSpaceEpochHandle stagingHandle;
+ private int ops = 0;
+ private long bytes = 0;
+ private long batchStartNanos = -1;
+
+ RestoreSession(RestoreMode mode, FlushListener flushListener, Logger logger) {
+ this.flushListener = flushListener;
+ this.logger = logger;
+ try {
+ stagingDir = Paths.get(spaceRootDir().getAbsolutePath(), UUID.randomUUID().toString()).toFile();
+ if (mode == RestoreMode.Overlay) {
+ active.get().checkpoint.createCheckpoint(stagingDir.toString());
+ } else {
+ Files.createDirectories(stagingDir.toPath());
+ }
+ stagingHandle = new RocksDBCPableKVSpaceEpochHandle(id, stagingDir, RocksDBCPableKVSpace.this.conf,
+ RocksDBCPableKVSpace.this::isRetired, logger, tags);
+ helper = new RocksDBKVSpaceWriterHelper(stagingHandle.db, writeOptions);
+ } catch (Throwable t) {
+ throw new KVEngineException("Begin restore failed", t);
+ }
+ }
+
+ private void ensureOpen() {
+ if (closed.get()) {
+ throw new IllegalStateException("Restore session already closed");
+ }
+ }
+
+ private void flushIfNeeded() {
+ if (adaptiveWriteBudget.shouldFlush(ops, bytes)) {
+ long start = batchStartNanos > 0 ? batchStartNanos : System.nanoTime();
+ helper.flush();
+ long latencyMillis = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
+ adaptiveWriteBudget.recordFlush(ops, bytes, latencyMillis);
+ flushListener.onFlush(ops, bytes);
+ ops = 0;
+ bytes = 0;
+ batchStartNanos = -1;
+ }
+ }
+
+ @Override
+ public IRestoreSession put(ByteString key, ByteString value) {
+ ensureOpen();
+ try {
+ helper.put(stagingHandle.cf, key, value);
+ if (ops == 0 && bytes == 0) {
+ batchStartNanos = System.nanoTime();
+ }
+ ops++;
+ bytes += key.size() + value.size();
+ flushIfNeeded();
+ return this;
+ } catch (RocksDBException e) {
+ throw new KVEngineException("Restore put failed", e);
+ }
+ }
+
+ @Override
+ public IRestoreSession metadata(ByteString metaKey, ByteString metaValue) {
+ ensureOpen();
+ try {
+ helper.metadata(stagingHandle.cf, metaKey, metaValue);
+ return this;
+ } catch (RocksDBException e) {
+ throw new KVEngineException("Restore metadata failed", e);
+ }
+ }
+
+ @Override
+ public void done() {
+ if (closed.compareAndSet(false, true)) {
+ try {
+ // Flush remaining data in current batch if any
+ if (ops > 0 || bytes > 0) {
+ long start = batchStartNanos > 0 ? batchStartNanos : System.nanoTime();
+ helper.flush();
+ long latencyMillis = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
+ adaptiveWriteBudget.recordFlush(ops, bytes, latencyMillis);
+ ops = 0;
+ bytes = 0;
+ batchStartNanos = -1;
+ }
+ helper.done();
+ // switch active to staging
+ switchTo(stagingHandle);
+ } catch (Throwable t) {
+ throw new KVEngineException("Restore done failed", t);
+ }
+ }
+ }
+
+ @Override
+ public void abort() {
+ if (closed.compareAndSet(false, true)) {
+ try {
+ helper.abort();
+ } catch (Throwable t) {
+ logger.warn("Abort restore session failed", t);
+ }
+ try {
+ stagingHandle.close();
+ } catch (Throwable t) {
+ logger.warn("Close staging RocksDB failed", t);
+ }
+ try {
+ deleteDir(stagingDir.toPath());
+ } catch (Throwable t) {
+ logger.warn("Delete staging dir failed: {}", stagingDir, t);
+ }
+ }
+ }
+
+ @Override
+ public int count() {
+ return helper.count();
+ }
+ }
+
+ private class MetricManager {
+ private final Gauge checkpointGauge; // hold a strong reference
+ private final Timer checkpointTimer;
+
+ MetricManager(String... metricTags) {
+ Tags tags = Tags.of(metricTags);
+ checkpointGauge = getGauge(id, GeneralKVSpaceMetric.CheckpointNumGauge, checkpoints::estimatedSize, tags);
+ checkpointTimer = KVSpaceMeters.getTimer(id, RocksDBKVSpaceMetric.CheckpointTimer, tags);
+ }
+
+ void close() {
+ checkpointGauge.close();
+ checkpointTimer.close();
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceEpochHandle.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceEpochHandle.java
new file mode 100644
index 000000000..faa5436fd
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceEpochHandle.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.IKVEngine.DEFAULT_NS;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBOptionsUtil.buildCPableCFDesc;
+
+import com.google.protobuf.Struct;
+import io.micrometer.core.instrument.Tags;
+import java.io.File;
+import java.lang.ref.Cleaner;
+import java.util.function.Predicate;
+import org.apache.bifromq.baseenv.EnvProvider;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.DBOptions;
+import org.slf4j.Logger;
+
+class RocksDBCPableKVSpaceEpochHandle extends RocksDBKVSpaceEpochHandle {
+ private static final Cleaner CLEANER = Cleaner.create(
+ EnvProvider.INSTANCE.newThreadFactory("kvspace-epoch-cleaner", true));
+ private final SpaceMetrics metrics;
+ private final Cleaner.Cleanable cleanable;
+
+ RocksDBCPableKVSpaceEpochHandle(String id,
+ File dir,
+ Struct conf,
+ Predicate<String> isRetired,
+ Logger logger,
+ Tags tags) {
+ super(dir, conf, logger);
+ this.metrics = new SpaceMetrics(id, db, dbOptions, cf, cfDesc.getOptions(),
+ tags.and("gen", dir.getName()), logger);
+ cleanable = CLEANER.register(this, new ClosableResources(id, dir.getName(), dbOptions, cfDesc, cf, db,
+ checkpoint, dir, isRetired, metrics, logger));
+ }
+
+ @Override
+ public void close() {
+ cleanable.clean();
+ }
+
+ @Override
+ protected DBOptions buildDBOptions(Struct conf) {
+ return RocksDBOptionsUtil.buildCPableDBOption(conf);
+ }
+
+ @Override
+ protected ColumnFamilyDescriptor buildCFDescriptor(Struct conf) {
+ return buildCPableCFDesc(DEFAULT_NS, conf);
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBDefaultConfigs.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBDefaultConfigs.java
new file mode 100644
index 000000000..4f3356f2f
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBDefaultConfigs.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.StructUtil.toValue;
+
+import com.google.protobuf.Struct;
+import org.apache.bifromq.baseenv.EnvProvider;
+import org.rocksdb.StatsLevel;
+import org.rocksdb.util.SizeUnit;
+
+/**
+ * Default configuration constants for RocksDB engines.
+ */
+public final class RocksDBDefaultConfigs {
+ public static final String DB_ROOT_DIR = "dbRootDir";
+ public static final String DB_CHECKPOINT_ROOT_DIR = "dbCheckpointRootDir";
+ public static final String ENABLE_STATS = "enableStats";
+ public static final String STATS_LEVEL = "statsLevel";
+ public static final String MANUAL_COMPACTION = "manualCompaction";
+ public static final String COMPACT_MIN_TOMBSTONE_KEYS = "compactMinTombstoneKeys";
+ public static final String COMPACT_MIN_TOMBSTONE_RANGES = "compactMinTombstoneRanges";
+ public static final String COMPACT_TOMBSTONE_RATIO = "compactTombstoneRatio";
+ public static final String BLOCK_CACHE_SIZE = "blockCacheSize";
+ public static final String WRITE_BUFFER_SIZE = "writeBufferSize";
+ public static final String MAX_WRITE_BUFFER_NUMBER = "maxWriteBufferNumber";
+ public static final String MIN_WRITE_BUFFER_NUMBER_TO_MERGE = "minWriteBufferNumberToMerge";
+ public static final String MIN_BLOB_SIZE = "minBlobSize";
+ public static final String INCREASE_PARALLELISM = "increaseParallelism";
+ public static final String MAX_BACKGROUND_JOBS = "maxBackgroundJobs";
+ public static final String LEVEL0_FILE_NUM_COMPACTION_TRIGGER = "level0FileNumCompactionTrigger";
+ public static final String LEVEL0_SLOWDOWN_WRITES_TRIGGER = "level0SlowdownWritesTrigger";
+ public static final String LEVEL0_STOP_WRITES_TRIGGER = "level0StopWritesTrigger";
+ public static final String MAX_BYTES_FOR_LEVEL_BASE = "maxBytesForLevelBase";
+ public static final String TARGET_FILE_SIZE_BASE = "targetFileSizeBase";
+ public static final String ASYNC_WAL_FLUSH = "asyncWALFlush";
+ public static final String FSYNC_WAL = "fsyncWAL";
+ public static final Struct CP;
+ public static final Struct WAL;
+
+ static {
+ // Build CP with all defaults
+ Struct.Builder configBuilder = Struct.newBuilder();
+ configBuilder.putFields(DB_ROOT_DIR, toValue(""));
+ configBuilder.putFields(ENABLE_STATS, toValue(false));
+ configBuilder.putFields(STATS_LEVEL, toValue(StatsLevel.EXCEPT_DETAILED_TIMERS.name()));
+ configBuilder.putFields(MANUAL_COMPACTION, toValue(true));
+ configBuilder.putFields(COMPACT_MIN_TOMBSTONE_KEYS, toValue(200000));
+ configBuilder.putFields(COMPACT_MIN_TOMBSTONE_RANGES, toValue(100000));
+ configBuilder.putFields(COMPACT_TOMBSTONE_RATIO, toValue(0.3));
+ configBuilder.putFields(BLOCK_CACHE_SIZE, toValue(32 * SizeUnit.MB));
+ configBuilder.putFields(WRITE_BUFFER_SIZE, toValue(128 * SizeUnit.MB));
+ configBuilder.putFields(MAX_WRITE_BUFFER_NUMBER, toValue(6));
+ configBuilder.putFields(MIN_WRITE_BUFFER_NUMBER_TO_MERGE, toValue(2));
+ configBuilder.putFields(MIN_BLOB_SIZE, toValue(2 * SizeUnit.KB));
+ configBuilder.putFields(INCREASE_PARALLELISM,
+ toValue(Math.max(EnvProvider.INSTANCE.availableProcessors() / 4, 2)));
+ configBuilder.putFields(MAX_BACKGROUND_JOBS,
+ toValue(Math.max(EnvProvider.INSTANCE.availableProcessors() / 4, 2)));
+ configBuilder.putFields(LEVEL0_FILE_NUM_COMPACTION_TRIGGER, toValue(8));
+ configBuilder.putFields(LEVEL0_SLOWDOWN_WRITES_TRIGGER, toValue(20));
+ configBuilder.putFields(LEVEL0_STOP_WRITES_TRIGGER, toValue(24));
+ configBuilder.putFields(MAX_BYTES_FOR_LEVEL_BASE, toValue(128 * 2 * 8 * SizeUnit.MB));
+ configBuilder.putFields(TARGET_FILE_SIZE_BASE, toValue(128 * 2 * SizeUnit.MB));
+ Struct sharedConfig = configBuilder.build();
+ CP = sharedConfig.toBuilder()
+ .putFields(DB_CHECKPOINT_ROOT_DIR, toValue(""))
+ .build();
+
+ // Build WAL based on CP
+ WAL = sharedConfig.toBuilder()
+ .putFields(ASYNC_WAL_FLUSH, toValue(true))
+ .putFields(FSYNC_WAL, toValue(false))
+ .build();
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBHelper.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBHelper.java
new file mode 100644
index 000000000..3d2b012cc
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBHelper.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static com.google.protobuf.UnsafeByteOperations.unsafeWrap;
+import static java.util.Collections.singletonList;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.DATA_SECTION_END;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.DATA_SECTION_START;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.META_SECTION_END;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.META_SECTION_START;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.fromMetaKey;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.toDataKey;
+import static org.apache.bifromq.basekv.utils.BoundaryUtil.compare;
+import static org.rocksdb.SizeApproximationFlag.INCLUDE_FILES;
+import static org.rocksdb.SizeApproximationFlag.INCLUDE_MEMTABLES;
+
+import com.google.protobuf.ByteString;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.DBOptions;
+import org.rocksdb.Range;
+import org.rocksdb.RocksDB;
+import org.rocksdb.Slice;
+
+class RocksDBHelper {
+ static RocksDBHandle openDBInDir(File dir, DBOptions dbOptions, ColumnFamilyDescriptor cfDesc) {
+ try {
+ List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
+ RocksDB db = RocksDB.open(dbOptions, dir.getAbsolutePath(), Collections.singletonList(cfDesc), cfHandles);
+ assert cfHandles.size() == 1;
+ ColumnFamilyHandle cf = cfHandles.get(0);
+ return new RocksDBHandle(db, cf);
+ } catch (Throwable e) {
+ throw new KVEngineException("Open RocksDB at dir failed", e);
+ }
+ }
+
+ static void deleteDir(Path path) throws IOException {
+ if (!path.toFile().exists()) {
+ return;
+ }
+ Files.walkFileTree(path, new SimpleFileVisitor<>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+ try {
+ Files.delete(file);
+ } catch (IOException e) {
+ // do nothing
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+ try {
+ Files.delete(dir);
+ } catch (IOException e) {
+ // do nothing
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ static long sizeOfBoundary(IRocksDBKVSpaceEpoch dbHandle, Boundary boundary) {
+ byte[] start =
+ !boundary.hasStartKey() ? DATA_SECTION_START : toDataKey(boundary.getStartKey().toByteArray());
+ byte[] end =
+ !boundary.hasEndKey() ? DATA_SECTION_END : toDataKey(boundary.getEndKey().toByteArray());
+ if (compare(start, end) < 0) {
+ try (Slice startSlice = new Slice(start); Slice endSlice = new Slice(end)) {
+ Range range = new Range(startSlice, endSlice);
+ return dbHandle.db()
+ .getApproximateSizes(dbHandle.cf(), singletonList(range), INCLUDE_MEMTABLES, INCLUDE_FILES)[0];
+ }
+ }
+ return 0;
+ }
+
+ static Map<ByteString, ByteString> getMetadata(IRocksDBKVSpaceEpoch dbHandle) {
+ try (RocksDBKVEngineIterator metaItr = new RocksDBKVEngineIterator(dbHandle.db(), dbHandle.cf(),
+ null,
+ META_SECTION_START,
+ META_SECTION_END)) {
+ Map<ByteString, ByteString> metaMap = new HashMap<>();
+ for (metaItr.seekToFirst(); metaItr.isValid(); metaItr.next()) {
+ metaMap.put(fromMetaKey(metaItr.key()), unsafeWrap(metaItr.value()));
+ }
+ return metaMap;
+ }
+ }
+
+ record RocksDBHandle(RocksDB db, ColumnFamilyHandle cf) {
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngine.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngine.java
similarity index 87%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngine.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngine.java
index f308963ae..0cd1190aa 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngine.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngine.java
@@ -19,9 +19,11 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.localengine.AbstractKVEngine;
-import org.apache.bifromq.basekv.localengine.KVEngineException;
+import static org.apache.bifromq.basekv.localengine.StructUtil.strVal;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
+
import com.google.common.base.Strings;
+import com.google.protobuf.Struct;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
@@ -33,20 +35,26 @@ import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.UUID;
import java.util.stream.Stream;
+import org.apache.bifromq.basekv.localengine.AbstractKVEngine;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.rocksdb.RocksDB;
+
+abstract class RocksDBKVEngine<T extends RocksDBKVSpace> extends AbstractKVEngine<T> {
+
+ public static final String IDENTITY_FILE = "IDENTITY";
+
+ static {
+ RocksDB.loadLibrary();
+ }
-abstract class RocksDBKVEngine<
- E extends RocksDBKVEngine<E, T, C>,
- T extends RocksDBKVSpace<E, T, C>,
- C extends RocksDBKVEngineConfigurator<C>
- > extends AbstractKVEngine<T, C> {
private final File dbRootDir;
private final String identity;
private final boolean isCreate;
private MetricManager metricManager;
- public RocksDBKVEngine(String overrideIdentity, C configurator) {
- super(overrideIdentity, configurator);
- dbRootDir = new File(configurator.dbRootDir());
+ public RocksDBKVEngine(String overrideIdentity, Struct confRoot) {
+ super(overrideIdentity, confRoot);
+ dbRootDir = new File(strVal(engineConf, DB_ROOT_DIR));
try {
Files.createDirectories(dbRootDir.getAbsoluteFile().toPath());
isCreate = isEmpty(dbRootDir.toPath());
@@ -103,7 +111,7 @@ abstract class RocksDBKVEngine<
private String loadIdentity(boolean create) {
try {
- Path identityFilePath = Paths.get(dbRootDir.getAbsolutePath(), "IDENTITY");
+ Path identityFilePath = Paths.get(dbRootDir.getAbsolutePath(), IDENTITY_FILE);
if (create) {
String identity =
Strings.isNullOrEmpty(overrideIdentity) ? UUID.randomUUID().toString() : overrideIdentity.trim();
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineIterator.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineIterator.java
similarity index 86%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineIterator.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineIterator.java
index cb4eb1c08..be15b6afb 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineIterator.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineIterator.java
@@ -19,7 +19,6 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import java.lang.ref.Cleaner;
import org.apache.bifromq.basekv.localengine.KVEngineException;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ReadOptions;
@@ -29,14 +28,10 @@ import org.rocksdb.Slice;
import org.rocksdb.Snapshot;
class RocksDBKVEngineIterator implements AutoCloseable {
- private static final Cleaner CLEANER = Cleaner.create();
private final RocksIterator rocksIterator;
- private final Cleaner.Cleanable onClose;
+ private final Runnable onClose;
- RocksDBKVEngineIterator(RocksDB db,
- ColumnFamilyHandle cfHandle,
- Snapshot snapshot,
- byte[] startKey,
+ RocksDBKVEngineIterator(RocksDB db, ColumnFamilyHandle cfHandle, Snapshot snapshot, byte[] startKey,
byte[] endKey) {
this(db, cfHandle, snapshot, startKey, endKey, true);
}
@@ -62,8 +57,7 @@ class RocksDBKVEngineIterator implements AutoCloseable {
readOptions.setSnapshot(snapshot);
}
rocksIterator = db.newIterator(cfHandle, readOptions);
- onClose = CLEANER.register(this, new NativeState(rocksIterator, readOptions, lowerSlice, upperSlice));
-
+ onClose = new NativeState(rocksIterator, readOptions, lowerSlice, upperSlice);
}
public byte[] key() {
@@ -102,9 +96,9 @@ class RocksDBKVEngineIterator implements AutoCloseable {
rocksIterator.seekForPrev(target);
}
- public void refresh() {
+ public void refresh(Snapshot snapshot) {
try {
- rocksIterator.refresh();
+ rocksIterator.refresh(snapshot);
} catch (Throwable e) {
throw new KVEngineException("Unable to refresh iterator", e);
}
@@ -112,7 +106,7 @@ class RocksDBKVEngineIterator implements AutoCloseable {
@Override
public void close() {
- onClose.clean();
+ onClose.run();
}
private record NativeState(RocksIterator itr, ReadOptions readOptions, Slice lowerSlice, Slice upperSlice)
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineProvider.java
similarity index 50%
copy from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
copy to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineProvider.java
index ca4c46187..3722d6a5a 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineProvider.java
@@ -19,27 +19,38 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
+import com.google.protobuf.Struct;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
import org.apache.bifromq.basekv.localengine.IKVEngine;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
-import java.nio.file.Paths;
-import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
+import org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider;
-public class RocksDBWALableKVEngineTest extends AbstractRocksDBKVEngineTest {
- protected RocksDBWALableKVEngineConfigurator configurator;
+/**
+ * Provider for RocksDB engine implementation.
+ */
+public class RocksDBKVEngineProvider implements IKVEngineProvider {
+ @Override
+ public Struct defaultsForCPable() {
+ return RocksDBDefaultConfigs.CP;
+ }
+
+ @Override
+ public Struct defaultsForWALable() {
+ return RocksDBDefaultConfigs.WAL;
+ }
+
+ @Override
+ public String type() {
+ return "rocksdb";
+ }
- @SneakyThrows
@Override
- protected void beforeStart() {
- super.beforeStart();
- String DB_NAME = "testDB";
- configurator = RocksDBWALableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .build();
+ public IKVEngine<? extends ICPableKVSpace> createCPable(String overrideIdentity, Struct conf) {
+ return new RocksDBCPableKVEngine(overrideIdentity, conf);
}
- @SneakyThrows
@Override
- protected IKVEngine<? extends IKVSpace> newEngine() {
- return new RocksDBWALableKVEngine(null, configurator);
+ public IKVEngine<? extends IWALableKVSpace> createWALable(String overrideIdentity, Struct conf) {
+ return new RocksDBWALableKVEngine(overrideIdentity, conf);
}
}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpace.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpace.java
new file mode 100644
index 000000000..adae463e9
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpace.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.StructUtil.boolVal;
+import static org.apache.bifromq.basekv.localengine.StructUtil.numVal;
+import static org.apache.bifromq.basekv.localengine.StructUtil.strVal;
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getCounter;
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getTimer;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.COMPACT_MIN_TOMBSTONE_KEYS;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.COMPACT_MIN_TOMBSTONE_RANGES;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.COMPACT_TOMBSTONE_RATIO;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.MANUAL_COMPACTION;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.deleteDir;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.getMetadata;
+
+import com.google.common.collect.Maps;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.Timer;
+import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.SneakyThrows;
+import org.apache.bifromq.baseenv.EnvProvider;
+import org.apache.bifromq.basekv.localengine.AbstractKVSpace;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.SyncContext;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.rocksdb.CompactRangeOptions;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+
+abstract class RocksDBKVSpace extends AbstractKVSpace<RocksDBKVSpaceEpochHandle> {
+
+ protected final Struct conf;
+ protected final RocksDBKVEngine<?> engine;
+ protected final ISyncContext syncContext;
+ protected final IWriteStatsRecorder writeStats;
+ private final File keySpaceDBDir;
+ private final ExecutorService compactionExecutor;
+ private final AtomicBoolean compacting;
+ private final ISyncContext.IRefresher metadataRefresher;
+ private SpaceMetrics spaceMetrics;
+ private volatile long lastCompactAt;
+ private volatile long nextCompactAt;
+
+ @SneakyThrows
+ public RocksDBKVSpace(String id,
+ Struct conf,
+ RocksDBKVEngine<?> engine,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ super(id, onDestroy, opMeters, logger, tags);
+ this.conf = conf;
+ this.engine = engine;
+ syncContext = new SyncContext();
+ metadataRefresher = syncContext.refresher();
+ compacting = new AtomicBoolean(false);
+ compactionExecutor = ExecutorServiceMetrics.monitor(Metrics.globalRegistry, new ThreadPoolExecutor(1, 1,
+ 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(),
+ EnvProvider.INSTANCE.newThreadFactory("kvspace-compactor-" + id)),
+ "compactor", "kvspace", Tags.of(tags));
+ if (boolVal(conf, MANUAL_COMPACTION)) {
+ int minKeys = (int) numVal(conf, COMPACT_MIN_TOMBSTONE_KEYS);
+ int minRanges = (int) numVal(conf, COMPACT_MIN_TOMBSTONE_RANGES);
+ double ratio = numVal(conf, COMPACT_TOMBSTONE_RATIO);
+ this.writeStats = new RocksDBKVSpaceCompactionTrigger(id, minKeys, minRanges, ratio, this::scheduleCompact,
+ tags);
+ } else {
+ this.writeStats = NoopWriteStatsRecorder.INSTANCE;
+ }
+ keySpaceDBDir = new File(strVal(conf, DB_ROOT_DIR), id);
+ }
+
+ @Override
+ protected void doOpen() {
+ spaceMetrics = new SpaceMetrics(tags);
+ reloadMetadata();
+ }
+
+ // Load metadata from DB and publish, without refresher gating
+ protected void reloadMetadata() {
+ updateMetadata(getMetadata(handle()));
+ }
+
+ protected void publishMetadata(Map<ByteString, ByteString> metadataUpdates) {
+ if (metadataUpdates.isEmpty()) {
+ return;
+ }
+ metadataRefresher.runIfNeeded((genBumped) -> {
+ Map<ByteString, ByteString> metaMap = Maps.newHashMap(currentMetadata());
+ metaMap.putAll(metadataUpdates);
+ updateMetadata(Collections.unmodifiableMap(metaMap));
+ });
+ }
+
+ @Override
+ protected void doClose() {
+ logger.debug("Close key range[{}]", id);
+ if (spaceMetrics != null) {
+ spaceMetrics.close();
+ }
+ }
+
+ @Override
+ protected void doDestroy() {
+ // Destroy the whole space root directory, including pointer file and all generations.
+ try {
+ if (keySpaceDBDir.exists()) {
+ deleteDir(keySpaceDBDir.toPath());
+ }
+ } catch (IOException e) {
+ logger.error("Failed to delete space root dir: {}", keySpaceDBDir, e);
+ }
+ }
+
+ protected File spaceRootDir() {
+ return keySpaceDBDir;
+ }
+
+ protected abstract RocksDBKVSpaceEpochHandle handle();
+
+ protected abstract WriteOptions writeOptions();
+
+ private void scheduleCompact() {
+ if (state() != State.Opening) {
+ return;
+ }
+ spaceMetrics.compactionSchedCounter.increment();
+ if (compacting.compareAndSet(false, true)) {
+ compactionExecutor.execute(spaceMetrics.compactionTimer.wrap(() -> {
+ logger.debug("KeyRange[{}] compaction start", id);
+ lastCompactAt = System.nanoTime();
+ writeStats.reset();
+ try (CompactRangeOptions options = new CompactRangeOptions()
+ .setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kSkip)
+ .setExclusiveManualCompaction(false)) {
+ synchronized (compacting) {
+ if (state() == State.Opening) {
+ IRocksDBKVSpaceEpoch handle = handle();
+ handle.db().compactRange(handle.cf(), null, null, options);
+ }
+ }
+ logger.debug("KeyRange[{}] compacted", id);
+ } catch (Throwable e) {
+ logger.error("KeyRange[{}] compaction error", id, e);
+ } finally {
+ compacting.set(false);
+ if (nextCompactAt > lastCompactAt) {
+ scheduleCompact();
+ }
+ }
+ }));
+ } else {
+ nextCompactAt = System.nanoTime();
+ }
+ }
+
+ @Override
+ protected long doSize(Boundary boundary) {
+ if (state() != State.Opening) {
+ return 0;
+ }
+ return RocksDBHelper.sizeOfBoundary(handle(), boundary);
+ }
+
+ private class SpaceMetrics {
+ private final Counter compactionSchedCounter;
+ private final Timer compactionTimer;
+
+ SpaceMetrics(Tags metricTags) {
+ compactionSchedCounter = getCounter(id, RocksDBKVSpaceMetric.ManualCompactionCounter, metricTags);
+ compactionTimer = getTimer(id, RocksDBKVSpaceMetric.ManualCompactionTimer, metricTags);
+ }
+
+ void close() {
+ compactionSchedCounter.close();
+ compactionTimer.close();
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpoint.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpoint.java
similarity index 66%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpoint.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpoint.java
index dbdb9c5ec..83b6e395e 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpoint.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpoint.java
@@ -19,10 +19,9 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import static com.google.protobuf.UnsafeByteOperations.unsafeWrap;
import static org.apache.bifromq.basekv.localengine.IKVEngine.DEFAULT_NS;
-import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.toMetaKey;
-import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVSpace.deleteDir;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.deleteDir;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.getMetadata;
import com.google.protobuf.ByteString;
import java.io.File;
@@ -30,14 +29,11 @@ import java.io.IOException;
import java.lang.ref.Cleaner;
import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
+import java.util.Map;
import java.util.function.Predicate;
-import java.util.function.Supplier;
-import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
-import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
import org.apache.bifromq.basekv.localengine.KVEngineException;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
-import org.apache.bifromq.basekv.proto.Boundary;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
@@ -46,14 +42,17 @@ import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
-class RocksDBKVSpaceCheckpoint extends RocksDBKVSpaceReader implements IRocksDBKVSpaceCheckpoint {
+class RocksDBKVSpaceCheckpoint implements IRocksDBKVSpaceCheckpoint {
private static final Cleaner CLEANER = Cleaner.create();
+ private final String id;
+ private final KVSpaceOpMeters opMeters;
+ private final Logger logger;
private final String cpId;
private final DBOptions dbOptions;
- private final RocksDB roDB;
private final ColumnFamilyDescriptor cfDesc;
- private final ColumnFamilyHandle cfHandle;
private final Cleaner.Cleanable cleanable;
+ private final IRocksDBKVSpaceEpoch handle;
+ private final Map<ByteString, ByteString> metadata;
RocksDBKVSpaceCheckpoint(String id,
String cpId,
@@ -61,7 +60,9 @@ class RocksDBKVSpaceCheckpoint extends RocksDBKVSpaceReader implements IRocksDBK
Predicate<String> isLatest,
KVSpaceOpMeters opMeters,
Logger logger) {
- super(id, opMeters, logger);
+ this.id = id;
+ this.opMeters = opMeters;
+ this.logger = logger;
this.cpId = cpId;
try {
cfDesc = new ColumnFamilyDescriptor(DEFAULT_NS.getBytes());
@@ -73,9 +74,20 @@ class RocksDBKVSpaceCheckpoint extends RocksDBKVSpaceReader implements IRocksDBK
List<ColumnFamilyDescriptor> cfDescs = List.of(cfDesc);
List<ColumnFamilyHandle> handles = new ArrayList<>();
- roDB = RocksDB.openReadOnly(dbOptions, cpDir.getAbsolutePath(), cfDescs, handles);
+ RocksDB roDB = RocksDB.openReadOnly(dbOptions, cpDir.getAbsolutePath(), cfDescs, handles);
- cfHandle = handles.get(0);
+ ColumnFamilyHandle cfHandle = handles.get(0);
+ handle = new IRocksDBKVSpaceEpoch() {
+ @Override
+ public RocksDB db() {
+ return roDB;
+ }
+
+ @Override
+ public ColumnFamilyHandle cf() {
+ return cfHandle;
+ }
+ };
cleanable = CLEANER.register(this, new ClosableResources(id,
cpId,
cpDir,
@@ -85,6 +97,7 @@ class RocksDBKVSpaceCheckpoint extends RocksDBKVSpaceReader implements IRocksDBK
dbOptions,
isLatest,
this.logger));
+ metadata = getMetadata(handle);
} catch (RocksDBException e) {
throw new KVEngineException("Failed to open checkpoint", e);
}
@@ -97,16 +110,8 @@ class RocksDBKVSpaceCheckpoint extends RocksDBKVSpaceReader implements IRocksDBK
}
@Override
- protected Optional<ByteString> doMetadata(ByteString metaKey) {
- try {
- byte[] valBytes = roDB.get(cfHandle(), toMetaKey(metaKey));
- if (valBytes == null) {
- return Optional.empty();
- }
- return Optional.of(unsafeWrap(valBytes));
- } catch (RocksDBException e) {
- throw new KVEngineException("Failed to read metadata", e);
- }
+ public IKVSpaceReader newReader() {
+ return new RocksDBKVSpaceCheckpointReader(id, opMeters, logger, this, handle, metadata);
}
@Override
@@ -114,42 +119,6 @@ class RocksDBKVSpaceCheckpoint extends RocksDBKVSpaceReader implements IRocksDBK
cleanable.clean();
}
- @Override
- protected RocksDB db() {
- return roDB;
- }
-
- @Override
- protected ColumnFamilyHandle cfHandle() {
- return cfHandle;
- }
-
- @Override
- protected ISyncContext.IRefresher newRefresher() {
- return new ISyncContext.IRefresher() {
-
- @Override
- public void runIfNeeded(Runnable runnable) {
- // no need to do any refresh, since it's readonly
- }
-
- @Override
- public <T> T call(Supplier<T> supplier) {
- return supplier.get();
- }
- };
- }
-
- @Override
- protected IKVSpaceIterator doNewIterator() {
- return new RocksDBKVSpaceIterator(db(), cfHandle(), null, Boundary.getDefaultInstance(), newRefresher(), false);
- }
-
- @Override
- protected IKVSpaceIterator doNewIterator(Boundary subBoundary) {
- return new RocksDBKVSpaceIterator(db(), cfHandle(), null, subBoundary, newRefresher(), false);
- }
-
private record ClosableResources(
String id,
String cpId,
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpointReader.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpointReader.java
new file mode 100644
index 000000000..9c689a848
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCheckpointReader.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.slf4j.Logger;
+
+class RocksDBKVSpaceCheckpointReader extends AbstractRocksDBKVSpaceReader {
+ private final IRocksDBKVSpaceEpoch checkpoint;
+ private final Map<ByteString, ByteString> metadata;
+ // Hold a strong reference to prevent premature GC of checkpoint
+ private RocksDBKVSpaceCheckpoint strongRef;
+
+ RocksDBKVSpaceCheckpointReader(String id,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ RocksDBKVSpaceCheckpoint strongRef,
+ IRocksDBKVSpaceEpoch checkpoint,
+ Map<ByteString, ByteString> metadata) {
+ super(id, opMeters, logger);
+ // keep strong reference during reader lifetime
+ this.strongRef = strongRef;
+ this.checkpoint = checkpoint;
+ this.metadata = metadata;
+ }
+
+ @Override
+ protected IRocksDBKVSpaceEpoch handle() {
+ return checkpoint;
+ }
+
+ @Override
+ protected RocksDBSnapshot snapshot() {
+ return new RocksDBSnapshot(this.handle(), null);
+ }
+
+ @Override
+ protected Optional<ByteString> doMetadata(ByteString metaKey) {
+ return Optional.ofNullable(metadata.get(metaKey));
+ }
+
+ @Override
+ protected IKVSpaceIterator doNewIterator(Boundary subBoundary) {
+ return new RocksDBKVSpaceIterator(snapshot(), subBoundary, new IteratorOptions(false, 52428));
+ }
+
+ @Override
+ public void close() {
+ // release strong reference to allow GC of checkpoint
+ strongRef = null;
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTrigger.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTrigger.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTrigger.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTrigger.java
index fe157e6e4..09c4ac2d4 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTrigger.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTrigger.java
@@ -19,11 +19,11 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters;
-import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Tags;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters;
+import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
class RocksDBKVSpaceCompactionTrigger implements IWriteStatsRecorder {
private final AtomicInteger totalKeyCount = new AtomicInteger();
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceEpochHandle.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceEpochHandle.java
new file mode 100644
index 000000000..c0a9afde8
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceEpochHandle.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getFunctionCounter;
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getFunctionTimer;
+import static org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters.getGauge;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.deleteDir;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBHelper.openDBInDir;
+
+import com.google.protobuf.Struct;
+import io.micrometer.core.instrument.FunctionCounter;
+import io.micrometer.core.instrument.FunctionTimer;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Tags;
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.Checkpoint;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.HistogramType;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.Statistics;
+import org.rocksdb.TickerType;
+import org.slf4j.Logger;
+
+abstract class RocksDBKVSpaceEpochHandle implements IRocksDBKVSpaceEpochHandle {
+ protected final Logger logger;
+ final DBOptions dbOptions;
+ final ColumnFamilyDescriptor cfDesc;
+ final RocksDB db;
+ final ColumnFamilyHandle cf;
+ final File dir;
+ final Checkpoint checkpoint;
+
+ RocksDBKVSpaceEpochHandle(File dir, Struct conf, Logger logger) {
+ this.dbOptions = buildDBOptions(conf);
+ this.cfDesc = buildCFDescriptor(conf);
+ RocksDBHelper.RocksDBHandle dbHandle = openDBInDir(dir, dbOptions, cfDesc);
+ this.db = dbHandle.db();
+ this.cf = dbHandle.cf();
+ this.dir = dir;
+ this.checkpoint = Checkpoint.create(db());
+ this.logger = logger;
+ }
+
+ @Override
+ public RocksDB db() {
+ return db;
+ }
+
+ @Override
+ public ColumnFamilyHandle cf() {
+ return cf;
+ }
+
+ protected abstract DBOptions buildDBOptions(Struct conf);
+
+ protected abstract ColumnFamilyDescriptor buildCFDescriptor(Struct conf);
+
+ protected record ClosableResources(String id,
+ String genId,
+ DBOptions dbOptions,
+ ColumnFamilyDescriptor cfDesc,
+ ColumnFamilyHandle cfHandle,
+ RocksDB db,
+ Checkpoint checkpoint,
+ File dir,
+ Predicate<String> isRetired,
+ SpaceMetrics metrics,
+ Logger log) implements Runnable {
+ @Override
+ public void run() {
+ // Ensure no metric suppliers call into RocksDB during close
+ try (AutoCloseable guard = metrics.beginClose()) {
+ metrics.close();
+ log.debug("Clean up generation[{}] of kvspace[{}]", genId, id);
+ // Close checkpoint before DB resources
+ checkpoint.close();
+ try {
+ db.destroyColumnFamilyHandle(cfHandle);
+ } catch (Throwable e) {
+ log.error("Failed to destroy column family handle of generation[{}] for kvspace[{}]", genId, id, e);
+ }
+ try {
+ db.close();
+ } catch (Throwable e) {
+ log.error("Failed to close RocksDB of generation[{}] for kvspace[{}]", genId, id, e);
+ }
+ cfDesc.getOptions().close();
+ dbOptions.close();
+ if (isRetired.test(genId)) {
+ log.debug("delete retired generation[{}] of kvspace[{}] in path: {}", genId, id,
+ dir.getAbsolutePath());
+ try {
+ deleteDir(dir.toPath());
+ } catch (Throwable e) {
+ log.error("Failed to clean retired generation at path:{}", dir, e);
+ }
+ }
+ } catch (Exception ignored) {
+ // ignore
+ }
+ }
+ }
+
+ protected static class SpaceMetrics {
+ private final ReentrantReadWriteLock rw = new ReentrantReadWriteLock();
+ private final Logger logger;
+ private final Gauge blockCacheSizeGauge;
+ private final Gauge tableReaderSizeGauge;
+ private final Gauge memTableSizeGauges;
+ private final Gauge pinedMemorySizeGauge;
+ private final Gauge totalSSTFileSizeGauge;
+ private final Gauge liveSSTFileSizeGauge;
+ private final Gauge liveDataSizeGauge;
+ private final Gauge estimateNumKeysGauge;
+ private final Gauge pendingCompactionBytesGauge;
+ private final Gauge numRunningCompactionsGauge;
+ private final Gauge numRunningFlushesGauge;
+ private final Gauge compactionPendingGauge;
+ private final Gauge memtableFlushPendingGauge;
+ private final Gauge backgroundErrorsGauge;
+ private final FunctionCounter bytesReadCounter;
+ private final FunctionCounter bytesWrittenCounter;
+ private final FunctionCounter blockCacheHitCounter;
+ private final FunctionCounter blockCacheMissCounter;
+ private final FunctionCounter blobCacheHitCounter;
+ private final FunctionCounter blobCacheMissCounter;
+ private final FunctionCounter bloomUsefulCounter;
+ private final FunctionTimer getLatencyTimer;
+ private final FunctionTimer writeLatencyTimer;
+ private final FunctionTimer seekLatencyTimer;
+ private final FunctionTimer blobGetLatencyTimer;
+ private final FunctionTimer blobWriteLatencyTimer;
+ private final FunctionTimer sstReadLatencyTimer;
+ private final FunctionTimer sstWriteLatencyTimer;
+ private final FunctionTimer flushLatencyTimer;
+ private final FunctionTimer compactionLatencyTimer;
+ private final FunctionTimer writeStallTimer;
+ private final Statistics statistics;
+ private volatile boolean closed = false;
+
+ SpaceMetrics(String id,
+ RocksDB db,
+ DBOptions dbOptions,
+ ColumnFamilyHandle cfHandle,
+ ColumnFamilyOptions cfOptions,
+ Tags metricTags,
+ Logger logger) {
+ this.logger = logger;
+ this.statistics = dbOptions.statistics();
+ blockCacheSizeGauge = getGauge(id, RocksDBKVSpaceMetric.BlockCache, () -> {
+ BlockBasedTableConfig cfg = (BlockBasedTableConfig) cfOptions.tableFormatConfig();
+ if (!cfg.noBlockCache()) {
+ return safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.block-cache-usage"));
+ }
+ return 0L;
+ }, metricTags);
+ tableReaderSizeGauge = getGauge(id, RocksDBKVSpaceMetric.TableReader,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.estimate-table-readers-mem")), metricTags);
+ memTableSizeGauges = getGauge(id, RocksDBKVSpaceMetric.MemTable,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.cur-size-all-mem-tables")), metricTags);
+ pinedMemorySizeGauge = getGauge(id, RocksDBKVSpaceMetric.PinnedMem, () -> {
+ BlockBasedTableConfig cfg = (BlockBasedTableConfig) cfOptions.tableFormatConfig();
+ if (!cfg.noBlockCache()) {
+ return safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.block-cache-pinned-usage"));
+ }
+ return 0L;
+ }, metricTags);
+ totalSSTFileSizeGauge = getGauge(id, RocksDBKVSpaceMetric.StateTotalSSTSize,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.total-sst-files-size")), metricTags);
+ liveSSTFileSizeGauge = getGauge(id, RocksDBKVSpaceMetric.StateLiveSSTSize,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.live-sst-files-size")), metricTags);
+ liveDataSizeGauge = getGauge(id, RocksDBKVSpaceMetric.StateLiveDataSize,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.estimate-live-data-size")), metricTags);
+ estimateNumKeysGauge = getGauge(id, RocksDBKVSpaceMetric.StateEstimateNumKeys,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.estimate-num-keys")), metricTags);
+ pendingCompactionBytesGauge = getGauge(id, RocksDBKVSpaceMetric.StatePendingCompactionBytes,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.estimate-pending-compaction-bytes")),
+ metricTags);
+ numRunningCompactionsGauge = getGauge(id, RocksDBKVSpaceMetric.StateRunningCompactions,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.num-running-compactions")), metricTags);
+ numRunningFlushesGauge = getGauge(id, RocksDBKVSpaceMetric.StateRunningFlushes,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.num-running-flushes")), metricTags);
+ compactionPendingGauge = getGauge(id, RocksDBKVSpaceMetric.StateCompactionPending,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.compaction-pending")), metricTags);
+ memtableFlushPendingGauge = getGauge(id, RocksDBKVSpaceMetric.StateMemTableFlushPending,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.mem-table-flush-pending")), metricTags);
+ backgroundErrorsGauge = getGauge(id, RocksDBKVSpaceMetric.StateBackgroundErrors,
+ () -> safeGet(() -> db.getLongProperty(cfHandle, "rocksdb.background-errors")), metricTags);
+
+ // Statistics-based meters
+ if (statistics != null) {
+ bytesReadCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.IOBytesReadCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) s.getTickerCount(TickerType.BYTES_READ)),
+ metricTags);
+ bytesWrittenCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.IOBytesWrittenCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) stats.getTickerCount(TickerType.BYTES_WRITTEN)),
+ metricTags);
+ blockCacheHitCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.BlockCacheHitCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) stats.getTickerCount(TickerType.BLOCK_CACHE_HIT)),
+ metricTags);
+ blockCacheMissCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.BlockCacheMissCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) stats.getTickerCount(TickerType.BLOCK_CACHE_MISS)),
+ metricTags);
+ blobCacheHitCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.BlobDBCacheHitCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) stats.getTickerCount(TickerType.BLOB_DB_CACHE_HIT)),
+ metricTags);
+ blobCacheMissCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.BlobDBCacheMissCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) stats.getTickerCount(TickerType.BLOB_DB_CACHE_MISS)),
+ metricTags);
+ bloomUsefulCounter = getFunctionCounter(id, RocksDBKVSpaceMetric.BloomUsefulCounter, statistics,
+ stats -> safeGetDouble(stats, (s) -> (double) stats.getTickerCount(TickerType.BLOOM_FILTER_USEFUL)),
+ metricTags);
+
+ getLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.GetLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.DB_GET).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.DB_GET).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ writeLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.WriteLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.DB_WRITE).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.DB_WRITE).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ seekLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.SeekLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.DB_SEEK).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.DB_SEEK).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ blobGetLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.GetLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.BLOB_DB_GET_MICROS).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.BLOB_DB_GET_MICROS).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ blobWriteLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.WriteLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.BLOB_DB_WRITE_MICROS).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.BLOB_DB_WRITE_MICROS).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ sstReadLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.SSTReadLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.SST_READ_MICROS).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.SST_READ_MICROS).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ sstWriteLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.SSTWriteLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.SST_WRITE_MICROS).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.SST_WRITE_MICROS).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ flushLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.FlushLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.FLUSH_TIME).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.FLUSH_TIME).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ compactionLatencyTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.CompactionLatency, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.COMPACTION_TIME).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.COMPACTION_TIME).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ writeStallTimer = getFunctionTimer(id, RocksDBKVSpaceMetric.WriteStall, statistics,
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.WRITE_STALL).getCount()),
+ stats -> safeGetLong(stats, s -> s.getHistogramData(HistogramType.WRITE_STALL).getSum()),
+ TimeUnit.MICROSECONDS,
+ metricTags);
+ } else {
+ bytesReadCounter = null;
+ bytesWrittenCounter = null;
+ blockCacheHitCounter = null;
+ blockCacheMissCounter = null;
+ blobCacheHitCounter = null;
+ blobCacheMissCounter = null;
+ bloomUsefulCounter = null;
+ getLatencyTimer = null;
+ writeLatencyTimer = null;
+ seekLatencyTimer = null;
+ blobGetLatencyTimer = null;
+ blobWriteLatencyTimer = null;
+ sstReadLatencyTimer = null;
+ sstWriteLatencyTimer = null;
+ flushLatencyTimer = null;
+ compactionLatencyTimer = null;
+ writeStallTimer = null;
+ }
+ }
+
+ private <T> double safeGetDouble(T obj, Function<T, Double> func) {
+ return safeGet(obj, func, 0d);
+ }
+
+ private <T> long safeGetLong(T obj, Function<T, Long> func) {
+ return safeGet(obj, func, 0L);
+ }
+
+ private <T, R> R safeGet(T obj, Function<T, R> func, R defVal) {
+ ReentrantReadWriteLock.ReadLock rl = rw.readLock();
+ rl.lock();
+ try {
+ if (closed) {
+ return defVal;
+ }
+ return func.apply(obj);
+ } catch (Throwable t) {
+ logger.warn("Unable to read RocksDB metric", t);
+ return defVal;
+ } finally {
+ rl.unlock();
+ }
+ }
+
+ private long safeGet(RocksDBLongGetter action) {
+ ReentrantReadWriteLock.ReadLock rl = rw.readLock();
+ rl.lock();
+ try {
+ if (closed) {
+ return 0L;
+ }
+ return action.get();
+ } catch (Throwable t) {
+ logger.warn("Unable to read RocksDB metric", t);
+ return 0L;
+ } finally {
+ rl.unlock();
+ }
+ }
+
+ AutoCloseable beginClose() {
+ ReentrantReadWriteLock.WriteLock wl = rw.writeLock();
+ wl.lock();
+ closed = true;
+ return wl::unlock;
+ }
+
+ void close() {
+ blockCacheSizeGauge.close();
+ memTableSizeGauges.close();
+ tableReaderSizeGauge.close();
+ pinedMemorySizeGauge.close();
+ totalSSTFileSizeGauge.close();
+ liveSSTFileSizeGauge.close();
+ liveDataSizeGauge.close();
+ estimateNumKeysGauge.close();
+ pendingCompactionBytesGauge.close();
+ numRunningCompactionsGauge.close();
+ numRunningFlushesGauge.close();
+ compactionPendingGauge.close();
+ memtableFlushPendingGauge.close();
+ backgroundErrorsGauge.close();
+ if (statistics != null) {
+ bytesReadCounter.close();
+ bytesWrittenCounter.close();
+ blockCacheHitCounter.close();
+ blockCacheMissCounter.close();
+ blobCacheHitCounter.close();
+ blobCacheMissCounter.close();
+ bloomUsefulCounter.close();
+ getLatencyTimer.close();
+ writeLatencyTimer.close();
+ seekLatencyTimer.close();
+ blobGetLatencyTimer.close();
+ blobWriteLatencyTimer.close();
+ sstReadLatencyTimer.close();
+ sstWriteLatencyTimer.close();
+ flushLatencyTimer.close();
+ compactionLatencyTimer.close();
+ writeStallTimer.close();
+ statistics.close();
+ }
+ }
+
+ interface RocksDBLongGetter {
+ long get() throws RocksDBException;
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceIterator.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceIterator.java
new file mode 100644
index 000000000..2b8c472da
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceIterator.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static com.google.protobuf.UnsafeByteOperations.unsafeWrap;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.DATA_SECTION_END;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.DATA_SECTION_START;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.fromDataKey;
+import static org.apache.bifromq.basekv.localengine.rocksdb.Keys.toDataKey;
+import static org.apache.bifromq.basekv.utils.BoundaryUtil.endKeyBytes;
+import static org.apache.bifromq.basekv.utils.BoundaryUtil.startKeyBytes;
+
+import com.google.protobuf.ByteString;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.Slice;
+
+@Slf4j
+class RocksDBKVSpaceIterator implements IKVSpaceIterator {
+ private final byte[] startKey;
+ private final byte[] endKey;
+ private final IteratorOptions options;
+ private final AtomicReference<RocksDBItrHolder> rocksItrHolder = new AtomicReference<>();
+ private final AtomicBoolean closed = new AtomicBoolean();
+ private final CloseListener closeListener;
+
+ public RocksDBKVSpaceIterator(RocksDBSnapshot snapshot, Boundary boundary, IteratorOptions options) {
+ this(snapshot, boundary, itr -> {
+ }, options);
+ }
+
+ public RocksDBKVSpaceIterator(RocksDBSnapshot snapshot,
+ Boundary boundary,
+ CloseListener closeListener,
+ IteratorOptions options) {
+ byte[] boundaryStartKey = startKeyBytes(boundary);
+ byte[] boundaryEndKey = endKeyBytes(boundary);
+ startKey = boundaryStartKey != null ? toDataKey(boundaryStartKey) : DATA_SECTION_START;
+ endKey = boundaryEndKey != null ? toDataKey(boundaryEndKey) : DATA_SECTION_END;
+ this.options = options;
+ this.closeListener = closeListener;
+ refresh(snapshot);
+ }
+
+ @Override
+ public ByteString key() {
+ return fromDataKey(rocksItrHolder.get().rocksIterator.key());
+ }
+
+ @Override
+ public ByteString value() {
+ return unsafeWrap(rocksItrHolder.get().rocksIterator.value());
+ }
+
+ @Override
+ public boolean isValid() {
+ return rocksItrHolder.get().rocksIterator.isValid();
+ }
+
+ @Override
+ public void next() {
+ rocksItrHolder.get().rocksIterator.next();
+ }
+
+ @Override
+ public void prev() {
+ rocksItrHolder.get().rocksIterator.prev();
+ }
+
+ @Override
+ public void seekToFirst() {
+ rocksItrHolder.get().rocksIterator.seekToFirst();
+ }
+
+ @Override
+ public void seekToLast() {
+ rocksItrHolder.get().rocksIterator.seekToLast();
+ }
+
+ @Override
+ public void seek(ByteString target) {
+ rocksItrHolder.get().rocksIterator.seek(toDataKey(target));
+ }
+
+ @Override
+ public void seekForPrev(ByteString target) {
+ rocksItrHolder.get().rocksIterator.seekForPrev(toDataKey(target));
+ }
+
+ public void refresh(RocksDBSnapshot snapshot) {
+ if (closed.get()) {
+ return;
+ }
+ try {
+ RocksDBItrHolder rocksItrHolder = this.rocksItrHolder.get();
+ if (rocksItrHolder == null) {
+ this.rocksItrHolder.set(build(snapshot));
+ } else if (rocksItrHolder.epoch == snapshot.epoch()) {
+ // same epoch, just refresh the snapshot
+ rocksItrHolder.rocksIterator.refresh(snapshot.snapshot());
+ } else {
+ this.rocksItrHolder.set(build(snapshot));
+ rocksItrHolder.close();
+ }
+ } catch (Throwable e) {
+ throw new KVEngineException("Unable to refresh iterator", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ if (closed.compareAndSet(false, true)) {
+ try {
+ RocksDBItrHolder rocksItrHolder = this.rocksItrHolder.getAndSet(null);
+ if (rocksItrHolder != null) {
+ rocksItrHolder.close();
+ }
+ } finally {
+ closeListener.onClose(this);
+ }
+ }
+ }
+
+ private RocksDBItrHolder build(RocksDBSnapshot snapshot) {
+ ReadOptions readOptions = new ReadOptions()
+ .setPinData(true)
+ .setFillCache(options.fillCache())
+ .setReadaheadSize(options.readAheadSize())
+ .setAutoPrefixMode(true);
+ Slice lowerSlice = new Slice(startKey);
+ readOptions.setIterateLowerBound(lowerSlice);
+ Slice upperSlice = new Slice(endKey);
+ readOptions.setIterateUpperBound(upperSlice);
+ if (snapshot != null) {
+ readOptions.setSnapshot(snapshot.snapshot());
+ }
+ RocksIterator rocksItr = snapshot.epoch().db().newIterator(snapshot.epoch().cf(), readOptions);
+ return new RocksDBItrHolder(snapshot.epoch(), rocksItr, readOptions, lowerSlice, upperSlice);
+ }
+
+ interface CloseListener {
+ void onClose(RocksDBKVSpaceIterator itr);
+ }
+
+ /**
+ * Holder for RocksDB iterator and its resources; close is idempotent.
+ */
+ private static final class RocksDBItrHolder {
+ final IRocksDBKVSpaceEpoch epoch;
+ final RocksIterator rocksIterator;
+ final ReadOptions readOptions;
+ final Slice lowerSlice;
+ final Slice upperSlice;
+ private final AtomicBoolean closed = new AtomicBoolean();
+
+ RocksDBItrHolder(IRocksDBKVSpaceEpoch epoch,
+ RocksIterator rocksIterator,
+ ReadOptions readOptions,
+ Slice lowerSlice,
+ Slice upperSlice) {
+ this.epoch = epoch;
+ this.rocksIterator = rocksIterator;
+ this.readOptions = readOptions;
+ this.lowerSlice = lowerSlice;
+ this.upperSlice = upperSlice;
+ }
+
+ public void close() {
+ // Ensure native resources are freed exactly once
+ if (closed.compareAndSet(false, true)) {
+ rocksIterator.close();
+ readOptions.close();
+ if (lowerSlice != null) {
+ lowerSlice.close();
+ }
+ if (upperSlice != null) {
+ upperSlice.close();
+ }
+ }
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceMigratableWriter.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceMigratableWriter.java
new file mode 100644
index 000000000..8c961eb76
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceMigratableWriter.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.function.Consumer;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceMigratableWriter;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+
+class RocksDBKVSpaceMigratableWriter extends RocksDBKVSpaceWriter
+ implements IKVSpaceMigratableWriter {
+
+ RocksDBKVSpaceMigratableWriter(String id,
+ IRocksDBKVSpaceEpochHandle dbHandle,
+ RocksDBKVEngine<?> engine,
+ WriteOptions writeOptions,
+ ISyncContext syncContext,
+ IWriteStatsRecorder.IRecorder writeStatsRecorder,
+ Consumer<Map<ByteString, ByteString>> afterWrite,
+ KVSpaceOpMeters opMeters,
+ Logger logger) {
+ super(id, dbHandle, engine, writeOptions, syncContext, writeStatsRecorder, afterWrite, opMeters, logger);
+ }
+
+ @Override
+ public IRestoreSession migrateTo(String targetSpaceId, Boundary boundary) {
+ try {
+ RocksDBCPableKVSpace targetKVSpace = (RocksDBCPableKVSpace) engine.createIfMissing(targetSpaceId);
+ IRestoreSession targetSpaceRestoreSession = targetKVSpace.startRestore((count, bytes) ->
+ logger.debug("Migrate {} kv to space[{}] from space[{}]: startKey={}, endKey={}",
+ count, targetSpaceId, id, boundary.getStartKey().toStringUtf8(),
+ boundary.getEndKey().toStringUtf8()));
+ try (IKVSpaceIterator itr = new RocksDBKVSpaceIterator(new RocksDBSnapshot(dbHandle, null), boundary,
+ new IteratorOptions(false, 52428))) {
+ for (itr.seekToFirst(); itr.isValid(); itr.next()) {
+ targetSpaceRestoreSession.put(itr.key(), itr.value());
+ }
+ }
+ // clear moved data in left range
+ helper.clear(dbHandle.cf(), boundary);
+ return targetSpaceRestoreSession;
+ } catch (Throwable e) {
+ throw new KVEngineException("Delete range in batch failed", e);
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceReader.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceReader.java
new file mode 100644
index 000000000..2e710bde3
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceReader.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.utils.BoundaryUtil.isValid;
+
+import com.google.common.collect.Sets;
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.slf4j.Logger;
+
+class RocksDBKVSpaceReader extends AbstractRocksDBKVSpaceReader implements IKVSpaceRefreshableReader {
+ private final ISyncContext.IRefresher refresher;
+ private final Supplier<IRocksDBKVSpaceEpoch> dbSupplier;
+ private final Supplier<Map<ByteString, ByteString>> metadataSupplier;
+ private final AtomicReference<RocksDBSnapshot> snapshot = new AtomicReference<>();
+ private final Set<RocksDBKVSpaceIterator> openedIterators = Sets.newConcurrentHashSet();
+ private final IteratorOptions iteratorOptions;
+
+ RocksDBKVSpaceReader(String id,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ ISyncContext.IRefresher refresher,
+ Supplier<IRocksDBKVSpaceEpoch> dbSupplier,
+ Supplier<Map<ByteString, ByteString>> metadataSupplier,
+ IteratorOptions iteratorOptions) {
+ super(id, opMeters, logger);
+ this.refresher = refresher;
+ this.dbSupplier = dbSupplier;
+ this.metadataSupplier = metadataSupplier;
+ this.snapshot.set(RocksDBSnapshot.take(dbSupplier.get()));
+ this.iteratorOptions = iteratorOptions;
+ }
+
+ @Override
+ public void refresh() {
+ refresher.runIfNeeded((genBumped) -> {
+ snapshot.getAndSet(RocksDBSnapshot.take(dbSupplier.get())).release();
+ openedIterators.forEach(itr -> itr.refresh(snapshot.get()));
+ });
+ }
+
+ @Override
+ public void close() {
+ openedIterators.forEach(RocksDBKVSpaceIterator::close);
+ RocksDBSnapshot oldSnapshot = snapshot.getAndSet(null);
+ oldSnapshot.release();
+ }
+
+ @Override
+ protected IRocksDBKVSpaceEpoch handle() {
+ return dbSupplier.get();
+ }
+
+ @Override
+ protected RocksDBSnapshot snapshot() {
+ return snapshot.get();
+ }
+
+ @Override
+ protected Optional<ByteString> doMetadata(ByteString metaKey) {
+ return refresher.call(() -> {
+ Map<ByteString, ByteString> metaMap = metadataSupplier.get();
+ return Optional.ofNullable(metaMap.get(metaKey));
+ });
+ }
+
+ @Override
+ protected IKVSpaceIterator doNewIterator(Boundary subBoundary) {
+ assert isValid(subBoundary);
+ RocksDBKVSpaceIterator itr = new RocksDBKVSpaceIterator(snapshot(), subBoundary,
+ openedIterators::remove, iteratorOptions);
+ openedIterators.add(itr);
+ return itr;
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriter.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriter.java
new file mode 100644
index 000000000..a907d8c3f
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriter.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import com.google.protobuf.ByteString;
+import java.util.Map;
+import java.util.function.Consumer;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
+import org.apache.bifromq.basekv.localengine.ISyncContext;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+
+class RocksDBKVSpaceWriter implements IKVSpaceWriter {
+ protected final String id;
+ protected final KVSpaceOpMeters opMeters;
+ protected final Logger logger;
+ protected final RocksDBKVEngine<?> engine;
+ protected final RocksDBKVSpaceWriterHelper helper;
+ protected final IRocksDBKVSpaceEpochHandle dbHandle;
+ private final IWriteStatsRecorder.IRecorder writeStatsRecorder;
+
+ RocksDBKVSpaceWriter(String id,
+ IRocksDBKVSpaceEpochHandle dbHandle,
+ RocksDBKVEngine<?> engine,
+ WriteOptions writeOptions,
+ ISyncContext syncContext,
+ IWriteStatsRecorder.IRecorder writeStatsRecorder,
+ Consumer<Map<ByteString, ByteString>> afterWrite,
+ KVSpaceOpMeters opMeters,
+ Logger logger) {
+ this(id, dbHandle, engine, syncContext, new RocksDBKVSpaceWriterHelper(dbHandle.db(), writeOptions),
+ writeStatsRecorder, afterWrite, opMeters, logger);
+ }
+
+ private RocksDBKVSpaceWriter(String id,
+ IRocksDBKVSpaceEpochHandle dbHandle,
+ RocksDBKVEngine<?> engine,
+ ISyncContext syncContext,
+ RocksDBKVSpaceWriterHelper writerHelper,
+ IWriteStatsRecorder.IRecorder writeStatsRecorder,
+ Consumer<Map<ByteString, ByteString>> afterWrite,
+ KVSpaceOpMeters opMeters,
+ Logger logger) {
+ this.id = id;
+ this.opMeters = opMeters;
+ this.logger = logger;
+ this.dbHandle = dbHandle;
+ this.engine = engine;
+ this.helper = writerHelper;
+ this.writeStatsRecorder = writeStatsRecorder;
+ writerHelper.addMutator(syncContext.mutator());
+ writerHelper.addAfterWriteCallback(dbHandle.cf(), afterWrite);
+ }
+
+ @Override
+ public String id() {
+ return id;
+ }
+
+ @Override
+ public IKVSpaceWriter metadata(ByteString metaKey, ByteString metaValue) {
+ try {
+ helper.metadata(dbHandle.cf(), metaKey, metaValue);
+ return this;
+ } catch (RocksDBException e) {
+ throw new KVEngineException("Put in batch failed", e);
+ }
+ }
+
+ @Override
+ public IKVSpaceWriter insert(ByteString key, ByteString value) {
+ try {
+ helper.insert(dbHandle.cf(), key, value);
+ writeStatsRecorder.recordInsert();
+ return this;
+ } catch (RocksDBException e) {
+ throw new KVEngineException("Insert in batch failed", e);
+ }
+ }
+
+ @Override
+ public IKVSpaceWriter put(ByteString key, ByteString value) {
+ try {
+ helper.put(dbHandle.cf(), key, value);
+ writeStatsRecorder.recordPut();
+ return this;
+ } catch (RocksDBException e) {
+ throw new KVEngineException("Put in batch failed", e);
+ }
+ }
+
+ @Override
+ public IKVSpaceWriter delete(ByteString key) {
+ try {
+ helper.delete(dbHandle.cf(), key);
+ writeStatsRecorder.recordDelete();
+ return this;
+ } catch (RocksDBException e) {
+ throw new KVEngineException("Single delete in batch failed", e);
+ }
+ }
+
+ @Override
+ public IKVSpaceWriter clear() {
+ return clear(Boundary.getDefaultInstance());
+ }
+
+ @Override
+ public IKVSpaceWriter clear(Boundary boundary) {
+ try {
+ helper.clear(dbHandle.cf(), boundary);
+ writeStatsRecorder.recordDeleteRange();
+ } catch (Throwable e) {
+ throw new KVEngineException("Delete range in batch failed", e);
+ }
+ return this;
+ }
+
+ @Override
+ public void done() {
+ opMeters.batchWriteCallTimer.record(() -> {
+ try {
+ opMeters.writeBatchSizeSummary.record(helper.count());
+ helper.done();
+ writeStatsRecorder.stop();
+ } catch (Throwable e) {
+ logger.error("Write Batch commit failed", e);
+ throw new KVEngineException("Batch commit failed", e);
+ }
+ });
+ }
+
+ @Override
+ public void abort() {
+ helper.abort();
+ }
+
+ @Override
+ public int count() {
+ return helper.count();
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriterHelper.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriterHelper.java
similarity index 87%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriterHelper.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriterHelper.java
index ab39b7fdd..f486057d4 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriterHelper.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceWriterHelper.java
@@ -57,7 +57,7 @@ class RocksDBKVSpaceWriterHelper {
}
- void addMutators(ISyncContext.IMutator mutator) {
+ void addMutator(ISyncContext.IMutator mutator) {
mutators.add(mutator);
}
@@ -102,19 +102,14 @@ class RocksDBKVSpaceWriterHelper {
if (batch.count() == 0) {
return;
}
- if (hasPendingMetadata()) {
- throw new IllegalStateException("Flush is not allowed when metadata changes exist");
- }
- runInMutators(() -> {
- try {
- if (batch.count() > 0) {
- db.write(writeOptions, batch);
- batch.clear();
- }
- } catch (Throwable e) {
- throw new KVEngineException("Range write error", e);
+ try {
+ if (batch.count() > 0) {
+ db.write(writeOptions, batch);
+ batch.clear();
}
- });
+ } catch (Throwable e) {
+ throw new KVEngineException("Range write error", e);
+ }
}
void done() {
@@ -131,6 +126,7 @@ class RocksDBKVSpaceWriterHelper {
batch.close();
}
}
+ return false;
});
for (ColumnFamilyHandle columnFamilyHandle : afterWriteCallbacks.keySet()) {
Map<ByteString, ByteString> updatedMetadata = metadataChanges.get(columnFamilyHandle);
@@ -156,20 +152,20 @@ class RocksDBKVSpaceWriterHelper {
return metadataChanges.values().stream().anyMatch(map -> !map.isEmpty());
}
- private void runInMutators(Runnable runnable) {
+ private void runInMutators(ISyncContext.IMutation mutation) {
if (mutators.isEmpty()) {
- runnable.run();
+ mutation.mutate();
return;
}
- AtomicReference<Runnable> finalRun = new AtomicReference<>();
+ AtomicReference<ISyncContext.IMutation> finalRun = new AtomicReference<>();
for (ISyncContext.IMutator mutator : mutators) {
if (finalRun.get() == null) {
- finalRun.set(() -> mutator.run(runnable));
+ finalRun.set(() -> mutator.run(mutation));
} else {
- Runnable innerRun = finalRun.get();
+ ISyncContext.IMutation innerRun = finalRun.get();
finalRun.set(() -> mutator.run(innerRun));
}
}
- finalRun.get().run();
+ finalRun.get().mutate();
}
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineConfigurator.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBOptionsUtil.java
similarity index 55%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineConfigurator.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBOptionsUtil.java
index 6860e7820..96a5550c4 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVEngineConfigurator.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBOptionsUtil.java
@@ -14,109 +14,67 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basekv.localengine.rocksdb;
+import static org.apache.bifromq.basekv.localengine.StructUtil.boolVal;
+import static org.apache.bifromq.basekv.localengine.StructUtil.numVal;
+import static org.apache.bifromq.basekv.localengine.StructUtil.strVal;
import static org.apache.bifromq.basekv.localengine.rocksdb.AutoCleaner.autoRelease;
-import static java.lang.Math.max;
-
-import org.apache.bifromq.baseenv.EnvProvider;
-import org.apache.bifromq.basekv.localengine.IKVEngineConfigurator;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-import lombok.experimental.SuperBuilder;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.ASYNC_WAL_FLUSH;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.BLOCK_CACHE_SIZE;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.ENABLE_STATS;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.INCREASE_PARALLELISM;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.LEVEL0_FILE_NUM_COMPACTION_TRIGGER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.LEVEL0_SLOWDOWN_WRITES_TRIGGER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.LEVEL0_STOP_WRITES_TRIGGER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.MAX_BACKGROUND_JOBS;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.MAX_BYTES_FOR_LEVEL_BASE;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.MAX_WRITE_BUFFER_NUMBER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.MIN_BLOB_SIZE;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.MIN_WRITE_BUFFER_NUMBER_TO_MERGE;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.STATS_LEVEL;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.TARGET_FILE_SIZE_BASE;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.WRITE_BUFFER_SIZE;
+
+import com.google.protobuf.Struct;
+import java.util.EnumSet;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter;
+import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyOptions;
-import org.rocksdb.ColumnFamilyOptionsInterface;
import org.rocksdb.CompactionStyle;
import org.rocksdb.CompressionType;
import org.rocksdb.DBOptions;
-import org.rocksdb.DBOptionsInterface;
import org.rocksdb.DataBlockIndexType;
import org.rocksdb.Env;
+import org.rocksdb.HistogramType;
import org.rocksdb.IndexType;
import org.rocksdb.LRUCache;
-import org.rocksdb.MutableColumnFamilyOptionsInterface;
-import org.rocksdb.MutableDBOptionsInterface;
import org.rocksdb.PrepopulateBlobCache;
import org.rocksdb.RateLimiter;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatsLevel;
import org.rocksdb.util.SizeUnit;
-@NoArgsConstructor
-@Getter
-@Setter
-@Accessors(chain = true, fluent = true)
-@SuperBuilder(toBuilder = true)
-public abstract class RocksDBKVEngineConfigurator<T extends RocksDBKVEngineConfigurator<T>>
- implements IKVEngineConfigurator {
- private String dbRootDir;
- @Builder.Default
- private boolean heuristicCompaction = false;
- @Builder.Default
- private int compactMinTombstoneKeys = 50000;
- @Builder.Default
- private int compactMinTombstoneRanges = 10000;
- @Builder.Default
- private double compactTombstoneKeysRatio = 0.3;
- @Builder.Default
- private long blockCacheSize = 32 * SizeUnit.MB;
- @Builder.Default
- private long writeBufferSize = 128 * SizeUnit.MB;
- @Builder.Default
- private int maxWriteBufferNumber = 6;
- @Builder.Default
- private int minWriteBufferNumberToMerge = 2;
- @Builder.Default
- private long minBlobSize = 2 * SizeUnit.KB;
- @Builder.Default
- private int increaseParallelism = max(EnvProvider.INSTANCE.availableProcessors() / 4, 2);
- @Builder.Default
- private int maxBackgroundJobs = max(EnvProvider.INSTANCE.availableProcessors() / 4, 2);
- @Builder.Default
- private int level0FileNumCompactionTrigger = 8;
- @Builder.Default
- private int level0SlowdownWritesTrigger = 20;
- @Builder.Default
- private int level0StopWritesTrigger = 24;
- @Builder.Default
- private long maxBytesForLevelBase = 512 * SizeUnit.MB;
- @Builder.Default
- private long targetFileSizeBase = 64 * SizeUnit.MB;
-
- public DBOptions dbOptions() {
- DBOptions targetOption = new DBOptions();
- configDBOptions((DBOptionsInterface<DBOptions>) targetOption);
- configDBOptions((MutableDBOptionsInterface<DBOptions>) targetOption);
- // we don't need atomic flush in both use cases
- targetOption.setAtomicFlush(false);
- return targetOption;
- }
-
- public ColumnFamilyOptions cfOptions(String name) {
- ColumnFamilyOptions targetOption = new ColumnFamilyOptions();
- configCFOptions(name, (ColumnFamilyOptionsInterface<ColumnFamilyOptions>) targetOption);
- configCFOptions(name, (MutableColumnFamilyOptionsInterface<ColumnFamilyOptions>) targetOption);
- return targetOption;
- }
-
- protected void configDBOptions(DBOptionsInterface<DBOptions> targetOption) {
- targetOption
- .setEnv(Env.getDefault())
+/**
+ * Build RocksDB options from Struct configuration. Keys align with provider schema.
+ */
+final class RocksDBOptionsUtil {
+ private static DBOptions buildDBOptions(Struct conf) {
+ DBOptions opts = new DBOptions();
+ opts.setEnv(Env.getDefault())
.setCreateIfMissing(true)
.setCreateMissingColumnFamilies(true)
.setAvoidUnnecessaryBlockingIO(true)
.setMaxManifestFileSize(64 * SizeUnit.MB)
- // log file settings
- .setRecycleLogFileNum(4)
+ // info log file settings
.setMaxLogFileSize(128 * SizeUnit.MB)
.setKeepLogFileNum(4)
// wal file settings
+ .setRecycleLogFileNum(4)
.setWalSizeLimitMB(0)
.setWalTtlSeconds(0)
.setEnablePipelinedWrite(true)
@@ -124,25 +82,69 @@ public abstract class RocksDBKVEngineConfigurator<T extends RocksDBKVEngineConfi
.setRateLimiter(autoRelease(new RateLimiter(512 * SizeUnit.MB,
RateLimiter.DEFAULT_REFILL_PERIOD_MICROS,
RateLimiter.DEFAULT_FAIRNESS,
- RateLimiter.DEFAULT_MODE, true), targetOption));
- }
-
- protected void configDBOptions(MutableDBOptionsInterface<DBOptions> targetOption) {
- targetOption
+ RateLimiter.DEFAULT_MODE, true), opts))
.setMaxOpenFiles(256)
- .setIncreaseParallelism(increaseParallelism)
- .setMaxBackgroundJobs(maxBackgroundJobs);
- }
-
- protected void configCFOptions(String name, ColumnFamilyOptionsInterface<ColumnFamilyOptions> targetOption) {
- targetOption
+ .setIncreaseParallelism((int) numVal(conf, INCREASE_PARALLELISM))
+ .setMaxBackgroundJobs((int) numVal(conf, MAX_BACKGROUND_JOBS));
+ // Atomic flush not used in current scenarios
+ opts.setAtomicFlush(false);
+
+ if (boolVal(conf, ENABLE_STATS)) {
+ EnumSet<HistogramType> ignoreTypes = EnumSet.allOf(HistogramType.class);
+ ignoreTypes.remove(HistogramType.DB_GET);
+ ignoreTypes.remove(HistogramType.DB_WRITE);
+ ignoreTypes.remove(HistogramType.DB_SEEK);
+ ignoreTypes.remove(HistogramType.SST_READ_MICROS);
+ ignoreTypes.remove(HistogramType.SST_WRITE_MICROS);
+ ignoreTypes.remove(HistogramType.BLOB_DB_GET_MICROS);
+ ignoreTypes.remove(HistogramType.BLOB_DB_WRITE_MICROS);
+ ignoreTypes.remove(HistogramType.FLUSH_TIME);
+ ignoreTypes.remove(HistogramType.COMPACTION_TIME);
+ Statistics statistics = new Statistics(ignoreTypes);
+ String level = strVal(conf, STATS_LEVEL);
+ statistics.setStatsLevel(StatsLevel.valueOf(level));
+ opts.setStatistics(statistics);
+ }
+ return opts;
+ }
+
+ static DBOptions buildCPableDBOption(Struct conf) {
+ DBOptions dbOptions = buildDBOptions(conf);
+ dbOptions.setRecycleLogFileNum(0)
+ .setAllowConcurrentMemtableWrite(true)
+ .setBytesPerSync(1048576);
+ return dbOptions;
+ }
+
+ static DBOptions buildWALableDBOption(Struct conf) {
+ DBOptions dbOptions = buildDBOptions(conf);
+ dbOptions.setManualWalFlush(boolVal(conf, ASYNC_WAL_FLUSH))
+ .setBytesPerSync(1048576)
+ .setAllowConcurrentMemtableWrite(true);
+ return dbOptions;
+ }
+
+ static ColumnFamilyDescriptor buildCPableCFDesc(String name, Struct conf) {
+ ColumnFamilyOptions cfOptions = buildCFOptions(conf);
+ cfOptions.setCompressionType(CompressionType.NO_COMPRESSION);
+ return new ColumnFamilyDescriptor(name.getBytes(), cfOptions);
+ }
+
+ static ColumnFamilyDescriptor buildWAlableCFDesc(String name, Struct conf) {
+ return new ColumnFamilyDescriptor(name.getBytes(), buildCFOptions(conf));
+ }
+
+ private static ColumnFamilyOptions buildCFOptions(Struct conf) {
+ ColumnFamilyOptions cfOptions = new ColumnFamilyOptions();
+ cfOptions
+ // immutable options start
.setMergeOperatorName("uint64add")
.setTableFormatConfig(
new BlockBasedTableConfig() //
// Begin to use partitioned index filters
// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters#how-to-use-it
.setIndexType(IndexType.kTwoLevelIndexSearch) //
- .setFilterPolicy(autoRelease(new BloomFilter(16, false), targetOption))
+ .setFilterPolicy(autoRelease(new BloomFilter(16, false), cfOptions))
.setPartitionFilters(true) //
.setMetadataBlockSize(8 * SizeUnit.KB) //
.setCacheIndexAndFilterBlocks(true) //
@@ -155,27 +157,24 @@ public abstract class RocksDBKVEngineConfigurator<T extends RocksDBKVEngineConfi
.setDataBlockHashTableUtilRatio(0.75)
// End of partitioned index filters settings.
.setBlockSize(4 * SizeUnit.KB)//
- .setBlockCache(autoRelease(new LRUCache(blockCacheSize, 8), targetOption)))
+ .setBlockCache(autoRelease(new LRUCache((long) numVal(conf, BLOCK_CACHE_SIZE), 8), cfOptions)))
// https://github.com/facebook/rocksdb/pull/5744
.setForceConsistencyChecks(true)
.setCompactionStyle(CompactionStyle.LEVEL)
- .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY);
- }
-
- protected void configCFOptions(String name, MutableColumnFamilyOptionsInterface<ColumnFamilyOptions> targetOption) {
- targetOption
+ .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY)
+ // mutable options start
.setCompressionType(CompressionType.LZ4_COMPRESSION)
.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION)
// Flushing options:
// write_buffer_size sets the size of a single mem_table. Once mem_table exceeds
// this size, it is marked immutable and a new one is created.
- .setWriteBufferSize(writeBufferSize)
+ .setWriteBufferSize((long) numVal(conf, WRITE_BUFFER_SIZE))
// Flushing options:
// max_write_buffer_number sets the maximum number of mem_tables, both active
// and immutable. If the active mem_table fills up and the total number of
// mem_tables is larger than max_write_buffer_number we stall further writes.
// This may happen if the flush process is slower than the write rate.
- .setMaxWriteBufferNumber(maxWriteBufferNumber)
+ .setMaxWriteBufferNumber((int) numVal(conf, MAX_WRITE_BUFFER_NUMBER))
// Flushing options:
// min_write_buffer_number_to_merge is the minimum number of mem_tables to be
// merged before flushing to storage. For example, if this option is set to 2,
@@ -185,20 +184,20 @@ public abstract class RocksDBKVEngineConfigurator<T extends RocksDBKVEngineConfi
// a single key. However, every Get() must traverse all immutable mem_tables
// linearly to check if the key is there. Setting this option too high may hurt
// read performance.
- .setMinWriteBufferNumberToMerge(minWriteBufferNumberToMerge)
+ .setMinWriteBufferNumberToMerge((int) numVal(conf, MIN_WRITE_BUFFER_NUMBER_TO_MERGE))
// Level Style Compaction:
// level0_file_num_compaction_trigger -- Once level 0 reaches this number of
// files, L0->L1 compaction is triggered. We can therefore estimate level 0
// size in stable state as
// write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger.
- .setLevel0FileNumCompactionTrigger(level0FileNumCompactionTrigger)
+ .setLevel0FileNumCompactionTrigger((int) numVal(conf, LEVEL0_FILE_NUM_COMPACTION_TRIGGER))
// Level Style Compaction:
// max_bytes_for_level_base and max_bytes_for_level_multiplier
// -- max_bytes_for_level_base is total size of level 1. As mentioned, we
// recommend that this be around the size of level 0. Each subsequent level
// is max_bytes_for_level_multiplier larger than previous one. The default
// is 10 and we do not recommend changing that.
- .setMaxBytesForLevelBase(maxBytesForLevelBase)
+ .setMaxBytesForLevelBase((long) numVal(conf, MAX_BYTES_FOR_LEVEL_BASE))
// Level Style Compaction:
// target_file_size_base and target_file_size_multiplier
// -- Files in level 1 will have target_file_size_base bytes. Each next
@@ -208,7 +207,7 @@ public abstract class RocksDBKVEngineConfigurator<T extends RocksDBKVEngineConfi
// number of database files, which is generally a good thing. We recommend setting
// target_file_size_base to be max_bytes_for_level_base / 10, so that there are
// 10 files in level 1.
- .setTargetFileSizeBase(targetFileSizeBase)
+ .setTargetFileSizeBase((long) numVal(conf, TARGET_FILE_SIZE_BASE))
// If prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
// create prefix bloom for memtable with the size of
// write_buffer_size * memtable_prefix_bloom_size_ratio.
@@ -217,58 +216,15 @@ public abstract class RocksDBKVEngineConfigurator<T extends RocksDBKVEngineConfi
// Soft limit on number of level-0 files. We start slowing down writes at this
// point. A value 0 means that no writing slow down will be triggered by number
// of files in level-0.
- .setLevel0SlowdownWritesTrigger(level0SlowdownWritesTrigger)
+ .setLevel0SlowdownWritesTrigger((int) numVal(conf, LEVEL0_SLOWDOWN_WRITES_TRIGGER))
// Maximum number of level-0 files. We stop writes at this point.
- .setLevel0StopWritesTrigger(level0StopWritesTrigger)
+ .setLevel0StopWritesTrigger((int) numVal(conf, LEVEL0_STOP_WRITES_TRIGGER))
.setLevelCompactionDynamicLevelBytes(false)
// enable blob files
.setEnableBlobFiles(true)
- .setMinBlobSize(minBlobSize())
+ .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY)
+ .setMinBlobSize((long) numVal(conf, MIN_BLOB_SIZE))
.enableBlobGarbageCollection();
- }
-
- public String dbRootDir() {
- return this.dbRootDir;
- }
-
- public boolean heuristicCompaction() {
- return this.heuristicCompaction;
- }
-
- public int compactMinTombstoneKeys() {
- return this.compactMinTombstoneKeys;
- }
-
- public int compactMinTombstoneRanges() {
- return this.compactMinTombstoneRanges;
- }
-
- public double compactTombstoneKeysRatio() {
- return this.compactTombstoneKeysRatio;
- }
-
- public T dbRootDir(String dbRootDir) {
- this.dbRootDir = dbRootDir;
- return (T) this;
- }
-
- public T heuristicCompaction(boolean heuristicCompaction) {
- this.heuristicCompaction = heuristicCompaction;
- return (T) this;
- }
-
- public T compactMinTombstoneKeys(int compactMinTombstoneKeys) {
- this.compactMinTombstoneKeys = compactMinTombstoneKeys;
- return (T) this;
- }
-
- public T compactMinTombstoneRanges(int compactMinTombstoneRanges) {
- this.compactMinTombstoneRanges = compactMinTombstoneRanges;
- return (T) this;
- }
-
- public T compactTombstoneKeysRatio(double compactTombstoneKeysRatio) {
- this.compactTombstoneKeysRatio = compactTombstoneKeysRatio;
- return (T) this;
+ return cfOptions;
}
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/IKVSpaceReader.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBSnapshot.java
similarity index 66%
copy from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/IKVSpaceReader.java
copy to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBSnapshot.java
index 69e2a6a0a..38b1dc274 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/IKVSpaceReader.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBSnapshot.java
@@ -17,18 +17,19 @@
* under the License.
*/
-package org.apache.bifromq.basekv.localengine;
+package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.proto.Boundary;
-import com.google.protobuf.ByteString;
-import java.util.Optional;
-public interface IKVSpaceReader extends IKVSpaceMetadata {
- boolean exist(ByteString key);
+import org.rocksdb.Snapshot;
- Optional<ByteString> get(ByteString key);
+record RocksDBSnapshot(IRocksDBKVSpaceEpoch epoch, Snapshot snapshot) {
+ static RocksDBSnapshot take(IRocksDBKVSpaceEpoch epoch) {
+ return new RocksDBSnapshot(epoch, epoch.db().getSnapshot());
+ }
- IKVSpaceIterator newIterator();
-
- IKVSpaceIterator newIterator(Boundary subBoundary);
+ void release() {
+ if (snapshot != null) {
+ epoch.db().releaseSnapshot(snapshot);
+ }
+ }
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngine.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngine.java
similarity index 58%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngine.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngine.java
index 288cbfa40..69b468d4c 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngine.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngine.java
@@ -19,23 +19,41 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
+import static org.apache.bifromq.basekv.localengine.StructUtil.strVal;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
+
+import com.google.protobuf.Struct;
+import java.nio.file.Paths;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.slf4j.Logger;
-public class RocksDBWALableKVEngine
- extends RocksDBKVEngine<RocksDBWALableKVEngine, RocksDBWALableKVSpace, RocksDBWALableKVEngineConfigurator> {
+class RocksDBWALableKVEngine extends RocksDBKVEngine<RocksDBWALableKVSpace> {
- public RocksDBWALableKVEngine(String overrideIdentity, RocksDBWALableKVEngineConfigurator configurator) {
- super(overrideIdentity, configurator);
+ RocksDBWALableKVEngine(String overrideIdentity, Struct conf) {
+ super(overrideIdentity, conf);
}
@Override
protected RocksDBWALableKVSpace doBuildKVSpace(String spaceId,
- RocksDBWALableKVEngineConfigurator configurator,
+ Struct conf,
Runnable onDestroy,
KVSpaceOpMeters opMeters,
Logger logger,
String... tags) {
- return new RocksDBWALableKVSpace(spaceId, configurator, this, onDestroy, opMeters, logger, tags);
+ return new RocksDBWALableKVSpace(spaceId, conf, this, onDestroy, opMeters, logger, tags);
+ }
+
+ @Override
+ protected Struct defaultConf() {
+ return RocksDBDefaultConfigs.WAL;
+ }
+
+ @Override
+ protected void validateSemantics(Struct conf) {
+ try {
+ Paths.get(strVal(conf, DB_ROOT_DIR));
+ } catch (Throwable t) {
+ throw new IllegalArgumentException("Invalid '" + DB_ROOT_DIR + "' path", t);
+ }
}
}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpace.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpace.java
similarity index 67%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpace.java
rename to base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpace.java
index cb37d7029..11e0fd192 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpace.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpace.java
@@ -19,16 +19,16 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.baseenv.EnvProvider;
-import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
-import org.apache.bifromq.basekv.localengine.KVEngineException;
-import org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters;
-import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
-import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
+import static org.apache.bifromq.basekv.localengine.StructUtil.boolVal;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.ASYNC_WAL_FLUSH;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.FSYNC_WAL;
+
+import com.google.protobuf.Struct;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;
+import java.nio.file.Files;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
@@ -36,30 +36,35 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bifromq.baseenv.EnvProvider;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
+import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
+import org.apache.bifromq.basekv.localengine.KVEngineException;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceMeters;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
import org.rocksdb.WriteOptions;
import org.slf4j.Logger;
-public class RocksDBWALableKVSpace
- extends RocksDBKVSpace<RocksDBWALableKVEngine, RocksDBWALableKVSpace, RocksDBWALableKVEngineConfigurator>
- implements IWALableKVSpace {
- private final RocksDBWALableKVEngineConfigurator configurator;
+class RocksDBWALableKVSpace extends RocksDBKVSpace implements IWALableKVSpace {
private final WriteOptions writeOptions;
private final AtomicReference<CompletableFuture<Long>> flushFutureRef = new AtomicReference<>();
private final ExecutorService flushExecutor;
private final MetricManager metricMgr;
+ private RocksDBWALableKVSpaceEpochHandle handle;
- public RocksDBWALableKVSpace(String id,
- RocksDBWALableKVEngineConfigurator configurator,
- RocksDBWALableKVEngine engine,
- Runnable onDestroy,
- KVSpaceOpMeters opMeters,
- Logger logger,
- String... tags) {
- super(id, configurator, engine, onDestroy, opMeters, logger, tags);
- this.configurator = configurator;
+ RocksDBWALableKVSpace(String id,
+ Struct conf,
+ RocksDBWALableKVEngine engine,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ super(id, conf, engine, onDestroy, opMeters, logger, tags);
writeOptions = new WriteOptions().setDisableWAL(false);
- if (!configurator.asyncWALFlush()) {
- writeOptions.setSync(configurator.fsyncWAL());
+ if (isSyncWALFlush()) {
+ writeOptions.setSync(isFsyncWAL());
}
flushExecutor = ExecutorServiceMetrics.monitor(Metrics.globalRegistry, new ThreadPoolExecutor(1, 1,
0L, TimeUnit.MILLISECONDS,
@@ -69,6 +74,23 @@ public class RocksDBWALableKVSpace
metricMgr = new MetricManager(tags);
}
+ @Override
+ protected void doOpen() {
+ try {
+ // WALable uses space root as DB directory
+ Files.createDirectories(spaceRootDir().getAbsoluteFile().toPath());
+ handle = new RocksDBWALableKVSpaceEpochHandle(id, spaceRootDir(), this.conf, logger, tags);
+ super.doOpen();
+ } catch (Throwable e) {
+ throw new KVEngineException("Failed to open WALable KVSpace", e);
+ }
+ }
+
+ @Override
+ protected RocksDBWALableKVSpaceEpochHandle handle() {
+ return handle;
+ }
+
@Override
protected void doClose() {
final CompletableFuture<Long> flushTaskFuture = Optional.ofNullable(flushFutureRef.get()).orElseGet(() -> {
@@ -84,6 +106,8 @@ public class RocksDBWALableKVSpace
}
writeOptions.close();
metricMgr.close();
+ // close handle
+ handle.close();
super.doClose();
}
@@ -97,7 +121,7 @@ public class RocksDBWALableKVSpace
if (state() != State.Opening) {
return CompletableFuture.failedFuture(new KVEngineException("KVSpace not open"));
}
- if (!configurator.asyncWALFlush()) {
+ if (isSyncWALFlush()) {
return CompletableFuture.completedFuture(System.nanoTime());
}
CompletableFuture<Long> flushFuture;
@@ -113,6 +137,12 @@ public class RocksDBWALableKVSpace
return flushFuture;
}
+ @Override
+ public IKVSpaceWriter toWriter() {
+ return new RocksDBKVSpaceWriter(id, handle, engine, writeOptions(), syncContext,
+ writeStats.newRecorder(), this::publishMetadata, opMeters, logger);
+ }
+
private void doFlush(CompletableFuture<Long> onDone) {
flushExecutor.submit(() -> {
long flashStartAt = System.nanoTime();
@@ -120,7 +150,7 @@ public class RocksDBWALableKVSpace
logger.trace("KVSpace[{}] flush wal start", id);
try {
Timer.Sample start = Timer.start();
- db.flushWal(configurator.fsyncWAL());
+ handle().db().flushWal(isFsyncWAL());
start.stop(metricMgr.flushTimer);
logger.trace("KVSpace[{}] flush complete", id);
} catch (Throwable e) {
@@ -136,11 +166,25 @@ public class RocksDBWALableKVSpace
});
}
+ private boolean isSyncWALFlush() {
+ return !boolVal(conf, ASYNC_WAL_FLUSH);
+ }
+
+ private boolean isFsyncWAL() {
+ return boolVal(conf, FSYNC_WAL);
+ }
+
+ @Override
+ public IKVSpaceRefreshableReader reader() {
+ return new RocksDBKVSpaceReader(id, opMeters, logger, syncContext.refresher(), this::handle,
+ this::currentMetadata, new IteratorOptions(false, 524288));
+ }
+
private class MetricManager {
private final Timer flushTimer;
MetricManager(String... metricTags) {
- flushTimer = KVSpaceMeters.getTimer(id, RocksDBKVSpaceMetric.FlushTimer, Tags.of(metricTags));
+ flushTimer = KVSpaceMeters.getTimer(id, RocksDBKVSpaceMetric.ManualFlushTimer, Tags.of(metricTags));
}
void close() {
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpaceEpochHandle.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpaceEpochHandle.java
new file mode 100644
index 000000000..301050d15
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpaceEpochHandle.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.IKVEngine.DEFAULT_NS;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBOptionsUtil.buildWALableDBOption;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBOptionsUtil.buildWAlableCFDesc;
+
+import com.google.protobuf.Struct;
+import io.micrometer.core.instrument.Tags;
+import java.io.File;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.DBOptions;
+import org.slf4j.Logger;
+
+class RocksDBWALableKVSpaceEpochHandle extends RocksDBKVSpaceEpochHandle {
+ private final SpaceMetrics metrics;
+ private final ClosableResources closableResources;
+
+ RocksDBWALableKVSpaceEpochHandle(String id, File dir, Struct conf, Logger logger, Tags tags) {
+ super(dir, conf, logger);
+ this.metrics = new SpaceMetrics(id, db, dbOptions, cf, cfDesc.getOptions(), tags.and("gen", "0"), logger);
+ closableResources = new ClosableResources(id, dir.getName(), dbOptions, cfDesc, cf, db, checkpoint, dir,
+ (test) -> false, metrics, logger);
+ }
+
+ @Override
+ public void close() {
+ closableResources.run();
+ }
+
+ @Override
+ protected DBOptions buildDBOptions(Struct conf) {
+ return buildWALableDBOption(conf);
+ }
+
+ @Override
+ protected ColumnFamilyDescriptor buildCFDescriptor(Struct conf) {
+ return buildWAlableCFDesc(DEFAULT_NS, conf);
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/metrics/RocksDBKVSpaceMetric.java b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/metrics/RocksDBKVSpaceMetric.java
new file mode 100644
index 000000000..8c3b796ff
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/java/org/apache/bifromq/basekv/localengine/rocksdb/metrics/RocksDBKVSpaceMetric.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb.metrics;
+
+import io.micrometer.core.instrument.Meter;
+import org.apache.bifromq.basekv.localengine.metrics.IKVSpaceMetric;
+
+/**
+ * RocksDB specific metrics for KVSpace.
+ */
+public enum RocksDBKVSpaceMetric implements IKVSpaceMetric {
+ BlockCache("basekv.le.rocksdb.mem.blockcache", Meter.Type.GAUGE),
+ TableReader("basekv.le.rocksdb.mem.tablereader", Meter.Type.GAUGE),
+ MemTable("basekv.le.rocksdb.mem.memtable", Meter.Type.GAUGE),
+ PinnedMem("basekv.le.rocksdb.mem.pinned", Meter.Type.GAUGE),
+ // DB state and capacity gauges
+ StateTotalSSTSize("basekv.le.rocksdb.state.totalsstsize", Meter.Type.GAUGE),
+ StateLiveSSTSize("basekv.le.rocksdb.state.livesstsize", Meter.Type.GAUGE),
+ StateLiveDataSize("basekv.le.rocksdb.state.livedatasize", Meter.Type.GAUGE),
+ StateEstimateNumKeys("basekv.le.rocksdb.state.keys.est", Meter.Type.GAUGE),
+ StatePendingCompactionBytes("basekv.le.rocksdb.state.pending.compaction.bytes", Meter.Type.GAUGE),
+ StateRunningCompactions("basekv.le.rocksdb.state.running.compactions", Meter.Type.GAUGE),
+ StateRunningFlushes("basekv.le.rocksdb.state.running.flushes", Meter.Type.GAUGE),
+ StateCompactionPending("basekv.le.rocksdb.state.pending.compactions", Meter.Type.GAUGE),
+ StateMemTableFlushPending("basekv.le.rocksdb.state.pending.flushes", Meter.Type.GAUGE),
+ StateBackgroundErrors("basekv.le.rocksdb.state.bg.errors", Meter.Type.GAUGE),
+ // IO and cache efficiency
+ IOBytesReadCounter("basekv.le.rocksdb.io.read.bytes", Meter.Type.COUNTER, true),
+ IOBytesWrittenCounter("basekv.le.rocksdb.io.write.bytes", Meter.Type.COUNTER, true),
+ BlockCacheHitCounter("basekv.le.rocksdb.block.cache.hit", Meter.Type.COUNTER, true),
+ BlockCacheMissCounter("basekv.le.rocksdb.block.cache.miss", Meter.Type.COUNTER, true),
+ BlobDBCacheHitCounter("basekv.le.rocksdb.blob.cache.hit", Meter.Type.COUNTER, true),
+ BlobDBCacheMissCounter("basekv.le.rocksdb.blob.cache.miss", Meter.Type.COUNTER, true),
+ BloomUsefulCounter("basekv.le.rocksdb.bloom.useful.count", Meter.Type.COUNTER, true),
+ CheckpointTimer("basekv.le.rocksdb.checkpoint.time", Meter.Type.TIMER),
+ TotalKeysGauge("basekv.le.rocksdb.compaction.keys", Meter.Type.GAUGE),
+ TotalTombstoneKeysGauge("basekv.le.rocksdb.compaction.delkeys", Meter.Type.GAUGE),
+ TotalTombstoneRangesGauge("basekv.le.rocksdb.compaction.delranges", Meter.Type.GAUGE),
+ ManualCompactionCounter("basekv.le.rocksdb.manual.compaction.count", Meter.Type.COUNTER),
+ ManualCompactionTimer("basekv.le.rocksdb.manual.compaction.time", Meter.Type.TIMER),
+ ManualFlushTimer("basekv.le.rocksdb.manual.flush.time", Meter.Type.TIMER),
+ // Histogram exposure: counters and gauges
+ GetLatency("basekv.le.rocksdb.latency.get.time", Meter.Type.TIMER, true),
+ WriteLatency("basekv.le.rocksdb.latency.write.time", Meter.Type.TIMER, true),
+ SeekLatency("basekv.le.rocksdb.latency.seek.time", Meter.Type.TIMER, true),
+ BlobGetLatency("basekv.le.rocksdb.latency.blob.get.time", Meter.Type.TIMER, true),
+ BlobWriteLatency("basekv.le.rocksdb.latency.blob.write.time", Meter.Type.TIMER, true),
+ SSTReadLatency("basekv.le.rocksdb.latency.sstread.time", Meter.Type.TIMER, true),
+ SSTWriteLatency("basekv.le.rocksdb.latency.sstwrite.time", Meter.Type.TIMER, true),
+ FlushLatency("basekv.le.rocksdb.latency.flush.time", Meter.Type.TIMER, true),
+ CompactionLatency("basekv.le.rocksdb.latency.compaction.time", Meter.Type.TIMER, true),
+ WriteStall("basekv.le.rocksdb.write.stall", Meter.Type.TIMER, true);
+
+ private final String metricName;
+ private final Meter.Type meterType;
+ private final boolean isFunction;
+
+ RocksDBKVSpaceMetric(String metricName, Meter.Type meterType) {
+ this(metricName, meterType, false);
+ }
+
+ RocksDBKVSpaceMetric(String metricName, Meter.Type meterType, boolean isFunction) {
+ this.metricName = metricName;
+ this.meterType = meterType;
+ this.isFunction = isFunction;
+ }
+
+ @Override
+ public String metricName() {
+ return metricName;
+ }
+
+ @Override
+ public Meter.Type meterType() {
+ return meterType;
+ }
+
+ @Override
+ public boolean isFunction() {
+ return isFunction;
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/main/resources/META-INF/services/org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider b/base-kv/base-kv-local-engine-rocksdb/src/main/resources/META-INF/services/org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider
new file mode 100644
index 000000000..a2c964ff1
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/main/resources/META-INF/services/org.apache.bifromq.basekv.localengine.spi.IKVEngineProvider
@@ -0,0 +1 @@
+org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVEngineProvider
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkState.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkState.java
similarity index 77%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkState.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkState.java
index e15796d8c..90d9c84e3 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkState.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkState.java
@@ -14,15 +14,16 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basekv.localengine.benchmark;
-import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVEngine;
-import org.apache.bifromq.basekv.localengine.KVEngineFactory;
-import org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVEngineConfigurator;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
+
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -31,6 +32,9 @@ import java.nio.file.Paths;
import java.util.Comparator;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.KVEngineFactory;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.TearDown;
@@ -49,11 +53,13 @@ abstract class BenchmarkState {
String DB_NAME = "testDB";
String DB_CHECKPOINT_DIR = "testDB_cp";
String uuid = UUID.randomUUID().toString();
- RocksDBCPableKVEngineConfigurator configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), uuid, DB_CHECKPOINT_DIR).toString())
- .dbRootDir(Paths.get(dbRootDir.toString(), uuid, DB_NAME).toString())
+ Struct conf = Struct.newBuilder()
+ .putFields(DB_ROOT_DIR,
+ Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), uuid, DB_NAME).toString()).build())
+ .putFields(DB_CHECKPOINT_ROOT_DIR,
+ Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), uuid, DB_CHECKPOINT_DIR).toString()).build())
.build();
- kvEngine = KVEngineFactory.createCPable(null, configurator);
+ kvEngine = KVEngineFactory.createCPable(null, "rocksdb", conf);
}
@Setup(Level.Trial)
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkTemplate.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkTemplate.java
similarity index 80%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkTemplate.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkTemplate.java
index aaf84c3f2..2ddfb7451 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkTemplate.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/BenchmarkTemplate.java
@@ -19,10 +19,11 @@
package org.apache.bifromq.basekv.localengine.benchmark;
-import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVEngine;
-import org.apache.bifromq.basekv.localengine.KVEngineFactory;
-import org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVEngineConfigurator;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
+
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -31,6 +32,9 @@ import java.nio.file.Paths;
import java.util.Comparator;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.KVEngineFactory;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.runner.Runner;
@@ -54,11 +58,13 @@ public abstract class BenchmarkTemplate {
String DB_NAME = "testDB";
String DB_CHECKPOINT_DIR = "testDB_cp";
String uuid = UUID.randomUUID().toString();
- RocksDBCPableKVEngineConfigurator configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), uuid, DB_CHECKPOINT_DIR).toString())
- .dbRootDir(Paths.get(dbRootDir.toString(), uuid, DB_NAME).toString())
+ Struct conf = Struct.newBuilder()
+ .putFields(DB_ROOT_DIR,
+ Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), uuid, DB_NAME).toString()).build())
+ .putFields(DB_CHECKPOINT_ROOT_DIR,
+ Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), uuid, DB_CHECKPOINT_DIR).toString()).build())
.build();
- kvEngine = KVEngineFactory.createCPable(null, configurator);
+ kvEngine = KVEngineFactory.createCPable(null, "rocksdb", conf);
kvEngine.start();
doSetup();
}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndGet_Benchmark.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndGet_Benchmark.java
similarity index 90%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndGet_Benchmark.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndGet_Benchmark.java
index 36c82a577..3876dd99a 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndGet_Benchmark.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndGet_Benchmark.java
@@ -22,15 +22,16 @@ package org.apache.bifromq.basekv.localengine.benchmark;
import static org.apache.bifromq.basekv.localengine.TestUtil.toByteString;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
-import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
-import org.apache.bifromq.basekv.proto.Boundary;
import com.google.protobuf.ByteString;
import java.util.Optional;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
+import org.apache.bifromq.basekv.proto.Boundary;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Group;
@@ -47,24 +48,16 @@ import org.openjdk.jmh.annotations.State;
public class ContinuousKeyDeleteAndGet_Benchmark extends BenchmarkTemplate {
private static int keyCount = 1000000;
private ByteString key = ByteString.copyFromUtf8("key");
- private IKVSpace kvSpace;
+ private ICPableKVSpace kvSpace;
+ private IKVSpaceRefreshableReader reader;
private IKVSpaceIterator itr;
private String rangeId = "testRange";
- @State(Scope.Thread)
- public static class BenchmarkThreadState {
- volatile int i = ThreadLocalRandom.current().nextInt(keyCount);
-
- @Setup(Level.Invocation)
- public void inc() {
- i = ThreadLocalRandom.current().nextInt(keyCount);
- }
- }
-
@Override
protected void doSetup() {
kvSpace = kvEngine.createIfMissing(rangeId);
- itr = kvSpace.newIterator();
+ reader = kvSpace.reader();
+ itr = reader.newIterator();
IKVSpaceWriter writer = kvSpace.toWriter();
for (int i = 0; i < keyCount; i++) {
writer.put(key.concat(toByteString(i)), ByteString.EMPTY);
@@ -88,6 +81,16 @@ public class ContinuousKeyDeleteAndGet_Benchmark extends BenchmarkTemplate {
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public Optional<ByteString> get(BenchmarkThreadState state) {
- return kvSpace.get(key.concat(toByteString(state.i)));
+ return reader.get(key.concat(toByteString(state.i)));
+ }
+
+ @State(Scope.Thread)
+ public static class BenchmarkThreadState {
+ volatile int i = ThreadLocalRandom.current().nextInt(keyCount);
+
+ @Setup(Level.Invocation)
+ public void inc() {
+ i = ThreadLocalRandom.current().nextInt(keyCount);
+ }
}
}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndSeek_Benchmark.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndSeek_Benchmark.java
similarity index 91%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndSeek_Benchmark.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndSeek_Benchmark.java
index dfd1123ea..26a83c83d 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndSeek_Benchmark.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeyDeleteAndSeek_Benchmark.java
@@ -22,14 +22,15 @@ package org.apache.bifromq.basekv.localengine.benchmark;
import static org.apache.bifromq.basekv.localengine.TestUtil.toByteString;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
-import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
-import org.apache.bifromq.basekv.proto.Boundary;
import com.google.protobuf.ByteString;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
+import org.apache.bifromq.basekv.proto.Boundary;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Group;
@@ -46,26 +47,18 @@ import org.openjdk.jmh.annotations.State;
public class ContinuousKeyDeleteAndSeek_Benchmark extends BenchmarkTemplate {
private static int keyCount = 1000000;
private static ByteString key = ByteString.copyFromUtf8("key");
- private IKVSpace kvSpace;
+ private ICPableKVSpace kvSpace;
+ private IKVSpaceRefreshableReader reader;
private IKVSpaceIterator itr;
private String rangeId = "testRange";
// IKVEngineIterator itr;
- @State(Scope.Thread)
- public static class BenchmarkThreadState {
- volatile int i = ThreadLocalRandom.current().nextInt(keyCount);
-
- @Setup(Level.Invocation)
- public void inc() {
- i = ThreadLocalRandom.current().nextInt(keyCount);
- }
- }
-
@Override
protected void doSetup() {
kvSpace = kvEngine.createIfMissing(rangeId);
- itr = kvSpace.newIterator();
+ reader = kvSpace.reader();
+ itr = reader.newIterator();
IKVSpaceWriter writer = kvSpace.toWriter();
for (int i = 0; i < keyCount; i++) {
writer.insert(key.concat(toByteString(i)), ByteString.copyFromUtf8("val" + i));
@@ -77,22 +70,13 @@ public class ContinuousKeyDeleteAndSeek_Benchmark extends BenchmarkTemplate {
writer.done();
}
-// @Benchmark
-// @Group("Seek")
-// @GroupThreads(1)
-// @BenchmarkMode(Mode.Throughput)
-// @OutputTimeUnit(TimeUnit.SECONDS)
-// public void delete(BenchmarkThreadState state) {
-// kvEngine.deleteRange(DEFAULT_NS, key.concat(toByteString(state.i)), key.concat(toByteString(state.i + 1)));
-// }
-
@Benchmark
@Group("Seek")
@GroupThreads(1)
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public boolean iterator(BenchmarkThreadState state) {
- IKVSpaceIterator itr = kvSpace.newIterator(Boundary.newBuilder()
+ IKVSpaceIterator itr = reader.newIterator(Boundary.newBuilder()
.setStartKey(key.concat(toByteString(state.i)))
.setEndKey(key.concat(toByteString(state.i + 1)))
.build());
@@ -101,4 +85,23 @@ public class ContinuousKeyDeleteAndSeek_Benchmark extends BenchmarkTemplate {
itr.seek(key.concat(toByteString(state.i)));
return itr.isValid();
}
+
+// @Benchmark
+// @Group("Seek")
+// @GroupThreads(1)
+// @BenchmarkMode(Mode.Throughput)
+// @OutputTimeUnit(TimeUnit.SECONDS)
+// public void delete(BenchmarkThreadState state) {
+// kvEngine.deleteRange(DEFAULT_NS, key.concat(toByteString(state.i)), key.concat(toByteString(state.i + 1)));
+// }
+
+ @State(Scope.Thread)
+ public static class BenchmarkThreadState {
+ volatile int i = ThreadLocalRandom.current().nextInt(keyCount);
+
+ @Setup(Level.Invocation)
+ public void inc() {
+ i = ThreadLocalRandom.current().nextInt(keyCount);
+ }
+ }
}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeek.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeek.java
similarity index 97%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeek.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeek.java
index 2594729cd..7aa89bd66 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeek.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeek.java
@@ -14,7 +14,7 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basekv.localengine.benchmark;
@@ -56,7 +56,7 @@ public class ContinuousKeySingleDeleteAndSeek {
// IKVEngineIterator itr = kvEngine.newIterator(DEFAULT_NS, key.concat(toByteString(state.i)),
// key.concat(toByteString(state.i + 1)));
// try (IKVEngineIterator itr = state.kvEngine.newIterator(DEFAULT_NS)) {
- state.itr.refresh();
+ state.reader.refresh();
// state.itr.seekToLast();
state.itr.seek(state.key.concat(toByteString(ThreadLocalRandom.current().nextInt(state.keyCount))));
bl.consume(state.itr.isValid());
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeekState.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeekState.java
similarity index 86%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeekState.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeekState.java
index 27ab0b4ed..0e421d4cf 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeekState.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/ContinuousKeySingleDeleteAndSeekState.java
@@ -22,10 +22,11 @@ package org.apache.bifromq.basekv.localengine.benchmark;
import static org.apache.bifromq.basekv.localengine.TestUtil.toByteString;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
+import com.google.protobuf.ByteString;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
-import com.google.protobuf.ByteString;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
@@ -33,14 +34,16 @@ import org.openjdk.jmh.annotations.State;
public class ContinuousKeySingleDeleteAndSeekState extends BenchmarkState {
int keyCount = 1000000;
ByteString key = ByteString.copyFromUtf8("key");
- private IKVSpace kvSpace;
+ IKVSpaceRefreshableReader reader;
IKVSpaceIterator itr;
+ private ICPableKVSpace kvSpace;
private String rangeId = "testRange";
@Override
protected void afterSetup() {
kvSpace = kvEngine.createIfMissing(rangeId);
- itr = kvSpace.newIterator();
+ reader = kvSpace.reader();
+ itr = reader.newIterator();
IKVSpaceWriter writer = kvSpace.toWriter();
for (int i = 0; i < keyCount; i++) {
@@ -49,7 +52,7 @@ public class ContinuousKeySingleDeleteAndSeekState extends BenchmarkState {
}
writer.put(key.concat(toByteString(keyCount)), ByteString.EMPTY);
writer.done();
- itr = kvSpace.newIterator();
+ itr = reader.newIterator();
}
@Override
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkload.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkload.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkload.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkload.java
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkloadState.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkloadState.java
similarity index 85%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkloadState.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkloadState.java
index 36e6a53ea..a0d9fb615 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkloadState.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/HybridWorkloadState.java
@@ -19,29 +19,31 @@
package org.apache.bifromq.basekv.localengine.benchmark;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
-import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
import com.google.protobuf.ByteString;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.IKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceIterator;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.IKVSpaceWriter;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
@Slf4j
@State(Scope.Benchmark)
public class HybridWorkloadState extends BenchmarkState {
+ ConcurrentHashMap<Long, IKVSpaceIterator> itrMap = new ConcurrentHashMap<>();
private String rangeId = "testRange";
private IKVSpace kvSpace;
+ private IKVSpaceRefreshableReader reader;
private IKVSpaceWriter writer;
- ConcurrentHashMap<Long, IKVSpaceIterator> itrMap = new ConcurrentHashMap<>();
-
@Override
protected void afterSetup() {
kvSpace = kvEngine.createIfMissing(rangeId);
+ reader = kvSpace.reader();
// itr = kvEngine.newIterator(rangeId);
}
@@ -68,24 +70,26 @@ public class HybridWorkloadState extends BenchmarkState {
}
public Optional<ByteString> randomGet() {
- return kvSpace.get(randomBS());
+ return reader.get(randomBS());
}
public boolean randomExist() {
- return kvSpace.exist(randomBS());
+ return reader.exist(randomBS());
}
public void seekToFirst() {
IKVSpaceIterator itr = itrMap.computeIfAbsent(Thread.currentThread().getId(),
- k -> kvSpace.newIterator());
- itr.refresh();
+ k -> reader.newIterator());
+ reader.refresh();
+ itr = itrMap.get(Thread.currentThread().getId());
itr.seekToFirst();
}
public void randomSeek() {
IKVSpaceIterator itr = itrMap.computeIfAbsent(Thread.currentThread().getId(),
- k -> kvSpace.newIterator());
- itr.refresh();
+ k -> reader.newIterator());
+ reader.refresh();
+ itr = itrMap.get(Thread.currentThread().getId());
itr.seek(randomBS());
}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGet.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGet.java
similarity index 92%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGet.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGet.java
index 5bdf7f398..60f2c49bf 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGet.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGet.java
@@ -26,6 +26,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Group;
@@ -56,7 +57,9 @@ public class SingleKeyUpdateAndGet {
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public void get(SingleKeyUpdateAndGetState state, Blackhole blackhole) {
- blackhole.consume(state.kvSpace.get(state.key).get());
+ try (IKVSpaceRefreshableReader reader = state.kvSpace.reader()) {
+ blackhole.consume(reader.get(state.key).get());
+ }
}
@Benchmark
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGetState.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGetState.java
similarity index 94%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGetState.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGetState.java
index 2dfd42d88..bdd06d90b 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGetState.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/benchmark/SingleKeyUpdateAndGetState.java
@@ -22,9 +22,9 @@ package org.apache.bifromq.basekv.localengine.benchmark;
import static org.apache.bifromq.basekv.localengine.TestUtil.toByteStringNativeOrder;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
@@ -32,9 +32,8 @@ import org.openjdk.jmh.annotations.State;
@State(Scope.Group)
public class SingleKeyUpdateAndGetState extends BenchmarkState {
ByteString key = ByteString.copyFromUtf8("key");
+ ICPableKVSpace kvSpace;
private String rangeId = "testRange";
- IKVSpace kvSpace;
-
@Override
protected void afterSetup() {
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/metrics/KVSpaceMetersTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/metrics/KVSpaceMetersTest.java
new file mode 100644
index 000000000..ed816d3c5
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/metrics/KVSpaceMetersTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.metrics;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.FunctionCounter;
+import io.micrometer.core.instrument.FunctionTimer;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.bifromq.basekv.localengine.rocksdb.metrics.RocksDBKVSpaceMetric;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class KVSpaceMetersTest {
+ private SimpleMeterRegistry registry;
+ private List<MeterRegistry> savedRegistries;
+ private String id;
+ private Tags baseTags;
+
+ @BeforeMethod
+ public void setUp() {
+ registry = new SimpleMeterRegistry();
+ savedRegistries = new ArrayList<>();
+ CompositeMeterRegistry composite = Metrics.globalRegistry;
+ savedRegistries.addAll(composite.getRegistries());
+ for (MeterRegistry r : savedRegistries) {
+ Metrics.removeRegistry(r);
+ }
+ Metrics.addRegistry(registry);
+ id = "kv1";
+ baseTags = Tags.of("env", "test");
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ Metrics.removeRegistry(registry);
+ registry.close();
+ for (MeterRegistry r : savedRegistries) {
+ Metrics.addRegistry(r);
+ }
+ }
+
+ @Test
+ public void counterIncrementAndUnregister() {
+ Counter c = KVSpaceMeters.getCounter(id, RocksDBKVSpaceMetric.ManualCompactionCounter, baseTags);
+ c.increment(2.0);
+ c.increment(3.0);
+ assertEquals(c.count(), 5.0, 0.0001);
+
+ assertFalse(registry
+ .find(RocksDBKVSpaceMetric.ManualCompactionCounter.metricName())
+ .tags("env", "test", "kvspace", id)
+ .meters().isEmpty());
+
+ c.close();
+ assertTrue(registry
+ .find(RocksDBKVSpaceMetric.ManualCompactionCounter.metricName())
+ .tags("env", "test", "kvspace", id)
+ .meters().isEmpty());
+ }
+
+ @Test
+ public void functionCounterBehaviorAndUnregister() {
+ class Holder {
+ double v;
+ }
+ Holder h = new Holder();
+ h.v = 7.0;
+
+ FunctionCounter fc = KVSpaceMeters.getFunctionCounter(
+ id, RocksDBKVSpaceMetric.IOBytesReadCounter, h, x -> x.v, baseTags);
+
+ assertEquals(fc.count(), 7.0, 0.0001);
+ h.v = 9.0;
+ assertEquals(fc.count(), 9.0, 0.0001);
+
+ assertFalse(registry
+ .find(RocksDBKVSpaceMetric.IOBytesReadCounter.metricName())
+ .tags("env", "test", "kvspace", id)
+ .meters().isEmpty());
+
+ fc.close();
+ assertTrue(registry
+ .find(RocksDBKVSpaceMetric.IOBytesReadCounter.metricName())
+ .tags("env", "test", "kvspace", id)
+ .meters().isEmpty());
+ }
+
+ @Test
+ public void functionTimerBehaviorAndUnregister() {
+ class T {
+ long c;
+ double t;
+ }
+ T h = new T();
+ h.c = 2L;
+ h.t = 1.5;
+
+ FunctionTimer ft = KVSpaceMeters.getFunctionTimer(
+ id, RocksDBKVSpaceMetric.GetLatency, h, x -> x.c, x -> x.t, TimeUnit.SECONDS, baseTags);
+
+ assertEquals(ft.count(), 2.0, 0.0001);
+ assertEquals(ft.totalTime(TimeUnit.SECONDS), 1.5, 0.0001);
+
+ h.c = 3L;
+ h.t = 2.5;
+ assertEquals(ft.count(), 3.0, 0.0001);
+ assertEquals(ft.totalTime(TimeUnit.SECONDS), 2.5, 0.0001);
+
+ assertFalse(registry
+ .find(RocksDBKVSpaceMetric.GetLatency.metricName())
+ .tags("env", "test", "kvspace", id)
+ .meters().isEmpty());
+
+ ft.close();
+ assertTrue(registry
+ .find(RocksDBKVSpaceMetric.GetLatency.metricName())
+ .tags("env", "test", "kvspace", id)
+ .meters().isEmpty());
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRawRocksDBTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRawRocksDBTest.java
similarity index 98%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRawRocksDBTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRawRocksDBTest.java
index c60398a3f..bdf3d15d2 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRawRocksDBTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRawRocksDBTest.java
@@ -14,17 +14,17 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
+ * under the License.
*/
package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.localengine.MockableTest;
-import org.apache.bifromq.basekv.localengine.TestUtil;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.MockableTest;
+import org.apache.bifromq.basekv.localengine.TestUtil;
import org.mockito.Mockito;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.Options;
@@ -35,10 +35,10 @@ abstract class AbstractRawRocksDBTest extends MockableTest {
RocksDB.loadLibrary();
}
- private Path dbRootDir;
- private Options options;
protected RocksDB db;
protected ColumnFamilyHandle cfHandle;
+ private Path dbRootDir;
+ private Options options;
@SneakyThrows
@Override
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVEngineTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBCPableEngineTest.java
similarity index 71%
copy from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVEngineTest.java
copy to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBCPableEngineTest.java
index 0de42463c..e5877a3ac 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBCPableEngineTest.java
@@ -25,17 +25,19 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
-import org.apache.bifromq.basekv.localengine.AbstractKVEngineTest;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
-import org.apache.bifromq.basekv.localengine.TestUtil;
import com.google.protobuf.ByteString;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.atomic.AtomicBoolean;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.AbstractCPableEngineTest;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.TestUtil;
import org.testng.annotations.Test;
-public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
+public abstract class AbstractRocksDBCPableEngineTest extends AbstractCPableEngineTest {
protected Path dbRootDir;
@SneakyThrows
@@ -66,12 +68,14 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
ByteString metaValue = ByteString.copyFromUtf8("metaValue");
ByteString key = ByteString.copyFromUtf8("key");
ByteString value = ByteString.copyFromUtf8("value");
- IKVSpace keyRange = engine.createIfMissing(rangeId);
+ ICPableKVSpace keyRange = engine.createIfMissing(rangeId);
keyRange.toWriter().put(key, value).metadata(metaKey, metaValue).done();
- assertTrue(keyRange.metadata(metaKey).isPresent());
- assertTrue(keyRange.metadata().blockingFirst().containsKey(metaKey));
- assertTrue(keyRange.exist(key));
- assertEquals(keyRange.get(key).get(), value);
+ try (IKVSpaceReader reader = keyRange.reader()) { // use reader for read APIs
+ assertTrue(reader.metadata(metaKey).isPresent());
+ assertTrue(keyRange.metadata().blockingFirst().containsKey(metaKey));
+ assertTrue(reader.exist(key));
+ assertEquals(reader.get(key).get(), value);
+ }
engine.stop();
engine = newEngine();
@@ -79,10 +83,12 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
assertEquals(engine.spaces().size(), 1);
IKVSpace keyRangeLoaded = engine.spaces().values().stream().findFirst().get();
assertEquals(keyRangeLoaded.id(), rangeId);
- assertTrue(keyRangeLoaded.metadata(metaKey).isPresent());
- assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
- assertTrue(keyRangeLoaded.exist(key));
- assertEquals(keyRangeLoaded.get(key).get(), value);
+ try (IKVSpaceReader reader = keyRangeLoaded.reader()) {
+ assertTrue(reader.metadata(metaKey).isPresent());
+ assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
+ assertTrue(reader.exist(key));
+ assertEquals(reader.get(key).get(), value);
+ }
// stop again and start
engine.stop();
@@ -91,10 +97,12 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
assertEquals(engine.spaces().size(), 1);
keyRangeLoaded = engine.spaces().values().stream().findFirst().get();
assertEquals(keyRangeLoaded.id(), rangeId);
- assertTrue(keyRangeLoaded.metadata(metaKey).isPresent());
- assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
- assertTrue(keyRangeLoaded.exist(key));
- assertEquals(keyRangeLoaded.get(key).get(), value);
+ try (IKVSpaceReader reader = keyRangeLoaded.reader()) {
+ assertTrue(reader.metadata(metaKey).isPresent());
+ assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
+ assertTrue(reader.exist(key));
+ assertEquals(reader.get(key).get(), value);
+ }
}
@Test
@@ -102,13 +110,15 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
String rangeId = "test_range1";
ByteString key = ByteString.copyFromUtf8("key");
ByteString value = ByteString.copyFromUtf8("value");
- IKVSpace keyRange = engine.createIfMissing(rangeId);
+ ICPableKVSpace keyRange = engine.createIfMissing(rangeId);
keyRange.toWriter().put(key, value).done();
engine.stop();
engine = newEngine();
engine.start();
keyRange = engine.createIfMissing(rangeId);
- assertTrue(keyRange.exist(key));
+ try (IKVSpaceReader reader = keyRange.reader()) {
+ assertTrue(reader.exist(key));
+ }
}
@Test
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVEngineTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBWALableEngineTest.java
similarity index 62%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVEngineTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBWALableEngineTest.java
index 0de42463c..f402a1db6 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AbstractRocksDBWALableEngineTest.java
@@ -19,23 +19,21 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import static org.apache.bifromq.basekv.localengine.rocksdb.AutoCleaner.autoRelease;
-import static org.awaitility.Awaitility.await;
import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
-import org.apache.bifromq.basekv.localengine.AbstractKVEngineTest;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
-import org.apache.bifromq.basekv.localengine.TestUtil;
import com.google.protobuf.ByteString;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.concurrent.atomic.AtomicBoolean;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.AbstractWALableEngineTest;
+import org.apache.bifromq.basekv.localengine.IKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
+import org.apache.bifromq.basekv.localengine.TestUtil;
import org.testng.annotations.Test;
-public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
+public abstract class AbstractRocksDBWALableEngineTest extends AbstractWALableEngineTest {
protected Path dbRootDir;
@SneakyThrows
@@ -66,12 +64,14 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
ByteString metaValue = ByteString.copyFromUtf8("metaValue");
ByteString key = ByteString.copyFromUtf8("key");
ByteString value = ByteString.copyFromUtf8("value");
- IKVSpace keyRange = engine.createIfMissing(rangeId);
+ IWALableKVSpace keyRange = engine.createIfMissing(rangeId);
keyRange.toWriter().put(key, value).metadata(metaKey, metaValue).done();
- assertTrue(keyRange.metadata(metaKey).isPresent());
- assertTrue(keyRange.metadata().blockingFirst().containsKey(metaKey));
- assertTrue(keyRange.exist(key));
- assertEquals(keyRange.get(key).get(), value);
+ try (IKVSpaceReader reader = keyRange.reader()) { // use reader for read APIs
+ assertTrue(reader.metadata(metaKey).isPresent());
+ assertTrue(keyRange.metadata().blockingFirst().containsKey(metaKey));
+ assertTrue(reader.exist(key));
+ assertEquals(reader.get(key).get(), value);
+ }
engine.stop();
engine = newEngine();
@@ -79,10 +79,12 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
assertEquals(engine.spaces().size(), 1);
IKVSpace keyRangeLoaded = engine.spaces().values().stream().findFirst().get();
assertEquals(keyRangeLoaded.id(), rangeId);
- assertTrue(keyRangeLoaded.metadata(metaKey).isPresent());
- assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
- assertTrue(keyRangeLoaded.exist(key));
- assertEquals(keyRangeLoaded.get(key).get(), value);
+ try (IKVSpaceReader reader = keyRangeLoaded.reader()) {
+ assertTrue(reader.metadata(metaKey).isPresent());
+ assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
+ assertTrue(reader.exist(key));
+ assertEquals(reader.get(key).get(), value);
+ }
// stop again and start
engine.stop();
@@ -91,10 +93,12 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
assertEquals(engine.spaces().size(), 1);
keyRangeLoaded = engine.spaces().values().stream().findFirst().get();
assertEquals(keyRangeLoaded.id(), rangeId);
- assertTrue(keyRangeLoaded.metadata(metaKey).isPresent());
- assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
- assertTrue(keyRangeLoaded.exist(key));
- assertEquals(keyRangeLoaded.get(key).get(), value);
+ try (IKVSpaceReader reader = keyRangeLoaded.reader()) {
+ assertTrue(reader.metadata(metaKey).isPresent());
+ assertTrue(keyRangeLoaded.metadata().blockingFirst().containsKey(metaKey));
+ assertTrue(reader.exist(key));
+ assertEquals(reader.get(key).get(), value);
+ }
}
@Test
@@ -102,33 +106,14 @@ public abstract class AbstractRocksDBKVEngineTest extends AbstractKVEngineTest {
String rangeId = "test_range1";
ByteString key = ByteString.copyFromUtf8("key");
ByteString value = ByteString.copyFromUtf8("value");
- IKVSpace keyRange = engine.createIfMissing(rangeId);
+ IWALableKVSpace keyRange = engine.createIfMissing(rangeId);
keyRange.toWriter().put(key, value).done();
engine.stop();
engine = newEngine();
engine.start();
keyRange = engine.createIfMissing(rangeId);
- assertTrue(keyRange.exist(key));
- }
-
- @Test
- public void autoReleaseTest() {
- Object owner = new Object();
- class Closeable implements AutoCloseable {
- AtomicBoolean closed = new AtomicBoolean();
-
- @Override
- public void close() {
- closed.set(true);
- }
+ try (IKVSpaceReader reader = keyRange.reader()) {
+ assertTrue(reader.exist(key));
}
- Closeable closeable = autoRelease(new Closeable(), owner);
- assertFalse(closeable.closed.get());
-
- owner = null;
- await().until(() -> {
- System.gc();
- return closeable.closed.get();
- });
}
}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AdaptiveWriteBudgetTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AdaptiveWriteBudgetTest.java
new file mode 100644
index 000000000..35dc385fe
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/AdaptiveWriteBudgetTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.testng.annotations.Test;
+
+public class AdaptiveWriteBudgetTest {
+
+ @Test
+ public void initialLimitsPositive() {
+ AdaptiveWriteBudget b = new AdaptiveWriteBudget();
+ assertTrue(b.currentEntryLimit() > 0);
+ assertTrue(b.currentByteLimit() > 0);
+ assertFalse(b.shouldFlush(0, 0));
+ assertTrue(b.shouldFlush(b.currentEntryLimit(), 0));
+ assertTrue(b.shouldFlush(0, b.currentByteLimit()));
+ }
+
+ @Test
+ public void fastRoundsIncreaseBudgets() {
+ AdaptiveWriteBudget b = new AdaptiveWriteBudget();
+ long e1 = b.currentEntryLimit();
+ long by1 = b.currentByteLimit();
+ for (int i = 0; i < 3; i++) {
+ b.recordFlush(e1, by1, 10); // fast
+ }
+ assertTrue(b.currentEntryLimit() >= e1);
+ assertTrue(b.currentByteLimit() >= by1);
+ }
+
+ @Test
+ public void slowRoundDecreasesBudgets() {
+ AdaptiveWriteBudget b = new AdaptiveWriteBudget();
+ for (int i = 0; i < 3; i++) {
+ b.recordFlush(b.currentEntryLimit(), b.currentByteLimit(), 10);
+ }
+ long beforeE = b.currentEntryLimit();
+ long beforeB = b.currentByteLimit();
+ b.recordFlush(beforeE, beforeB, 200); // slow
+ assertTrue(b.currentEntryLimit() <= beforeE);
+ assertTrue(b.currentByteLimit() <= beforeB);
+ }
+
+ @Test
+ public void emaAndClampBehavior() {
+ AdaptiveWriteBudget b = new AdaptiveWriteBudget();
+ for (int i = 0; i < 10; i++) {
+ b.recordFlush(100, 1024 * 1024, 30); // fast-ish
+ b.recordFlush(100, 1024 * 1024, 120); // slow-ish
+ }
+ assertTrue(b.currentEntryLimit() >= 1);
+ assertTrue(b.currentByteLimit() >= 1);
+ }
+
+ @Test
+ public void noopOnZeroInputs() {
+ AdaptiveWriteBudget b = new AdaptiveWriteBudget();
+ long e = b.currentEntryLimit();
+ long by = b.currentByteLimit();
+ b.recordFlush(0, 0, 30);
+ b.recordFlush(100, 1024, 0);
+ assertEquals(b.currentEntryLimit(), e);
+ assertEquals(b.currentByteLimit(), by);
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/KVSpaceCheckpointTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/KVSpaceCheckpointTest.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/KVSpaceCheckpointTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/KVSpaceCheckpointTest.java
index 429d53426..5499dc115 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/KVSpaceCheckpointTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/KVSpaceCheckpointTest.java
@@ -25,8 +25,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import org.apache.bifromq.basekv.localengine.TestUtil;
-import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import io.micrometer.core.instrument.Tags;
import java.io.File;
import java.lang.reflect.Method;
@@ -36,6 +34,8 @@ import java.nio.file.Paths;
import java.util.UUID;
import java.util.function.Predicate;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.TestUtil;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.mockito.Mock;
import org.rocksdb.Checkpoint;
import org.rocksdb.ColumnFamilyDescriptor;
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/OverrideIdentityTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/OverrideIdentityTest.java
similarity index 61%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/OverrideIdentityTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/OverrideIdentityTest.java
index f999f0163..4f316c4cc 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/OverrideIdentityTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/OverrideIdentityTest.java
@@ -19,13 +19,11 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
import static org.testng.Assert.assertEquals;
-import org.apache.bifromq.basekv.localengine.ICPableKVEngineConfigurator;
-import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVEngine;
-import org.apache.bifromq.basekv.localengine.KVEngineFactory;
-import org.apache.bifromq.basekv.localengine.MockableTest;
+import com.google.protobuf.Struct;
import java.io.File;
import java.lang.reflect.Method;
import java.nio.file.Files;
@@ -34,13 +32,18 @@ import java.nio.file.Paths;
import java.util.Comparator;
import java.util.UUID;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.KVEngineFactory;
+import org.apache.bifromq.basekv.localengine.MockableTest;
+import org.apache.bifromq.basekv.localengine.StructUtil;
import org.testng.annotations.Test;
public class OverrideIdentityTest extends MockableTest {
private final String DB_NAME = "testDB";
private final String DB_CHECKPOINT_DIR = "testDB_cp";
- private IKVEngine<? extends ICPableKVSpace> engine;
public Path dbRootDir;
+ private IKVEngine<? extends ICPableKVSpace> engine;
@SneakyThrows
protected void doSetup(Method method) {
@@ -58,33 +61,31 @@ public class OverrideIdentityTest extends MockableTest {
@Test
public void testOverrideIdentity() {
String overrideIdentity = UUID.randomUUID().toString();
- ICPableKVEngineConfigurator configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString())
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(DB_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()))
+ .putFields(DB_CHECKPOINT_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString()))
.build();
- engine = KVEngineFactory.createCPable(overrideIdentity, configurator);
+ engine = KVEngineFactory.createCPable(overrideIdentity, "rocksdb", conf);
engine.start();
assertEquals(engine.id(), overrideIdentity);
engine.stop();
// restart without overrideIdentity specified
- configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString())
+ conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(DB_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()))
+ .putFields(DB_CHECKPOINT_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString()))
.build();
-
- engine = KVEngineFactory.createCPable(null, configurator);
+ engine = KVEngineFactory.createCPable(null, "rocksdb", conf);
engine.start();
assertEquals(engine.id(), overrideIdentity);
engine.stop();
// restart with different overrideIdentity specified
String another = UUID.randomUUID().toString();
- configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString())
+ conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(DB_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()))
+ .putFields(DB_CHECKPOINT_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString()))
.build();
-
- engine = KVEngineFactory.createCPable(another, configurator);
+ engine = KVEngineFactory.createCPable(another, "rocksdb", conf);
engine.start();
assertEquals(engine.id(), overrideIdentity);
@@ -93,23 +94,21 @@ public class OverrideIdentityTest extends MockableTest {
@Test
public void testCanOnlyOverrideWhenInit() {
- ICPableKVEngineConfigurator configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString())
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(DB_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()))
+ .putFields(DB_CHECKPOINT_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString()))
.build();
-
- engine = KVEngineFactory.createCPable(null, configurator);
+ engine = KVEngineFactory.createCPable(null, "rocksdb", conf);
engine.start();
String identity = engine.id();
engine.stop();
// restart with overrideIdentity specified
String overrideIdentity = UUID.randomUUID().toString();
- configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString())
+ conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(DB_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()))
+ .putFields(DB_CHECKPOINT_ROOT_DIR, StructUtil.toValue(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString()))
.build();
-
- engine = KVEngineFactory.createCPable(overrideIdentity, configurator);
+ engine = KVEngineFactory.createCPable(overrideIdentity, "rocksdb", conf);
engine.start();
assertEquals(engine.id(), identity);
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngineTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngineTest.java
similarity index 74%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngineTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngineTest.java
index ffc201350..df0974467 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVEngineTest.java
@@ -19,37 +19,41 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBDefaultConfigs.DB_ROOT_DIR;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
-import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
-import org.apache.bifromq.basekv.localengine.IKVEngine;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import io.reactivex.rxjava3.disposables.Disposable;
import java.io.File;
import java.nio.file.Paths;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.IKVSpace;
import org.testng.annotations.Test;
-public class RocksDBCPableKVEngineTest extends AbstractRocksDBKVEngineTest {
+public class RocksDBCPableKVEngineTest extends AbstractRocksDBCPableEngineTest {
private final String DB_NAME = "testDB";
private final String DB_CHECKPOINT_DIR = "testDB_cp";
- private RocksDBCPableKVEngineConfigurator configurator;
+ private Struct conf;
@SneakyThrows
@Override
protected void beforeStart() {
super.beforeStart();
- configurator = RocksDBCPableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
- .dbCheckpointRootDir(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString())
+ conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(DB_ROOT_DIR, Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()).build())
+ .putFields(DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), DB_CHECKPOINT_DIR).toString()).build())
.build();
}
@SneakyThrows
@Override
protected IKVEngine<? extends ICPableKVSpace> newEngine() {
- return new RocksDBCPableKVEngine(null, configurator);
+ return new RocksDBCPableKVEngine(null, conf);
}
@Test
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceCleanupInactiveTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceCleanupInactiveTest.java
new file mode 100644
index 000000000..536497b79
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceCleanupInactiveTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVEngine.IDENTITY_FILE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceCleanupInactiveTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_cleanup_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testCleanInactiveOnStartup() throws Exception {
+ String spaceId = "space_cleanup";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Path spaceRoot = dbRoot.resolve(spaceId);
+ Files.createDirectories(spaceRoot);
+ Files.createDirectories(cpRoot);
+ // Ensure engine identity exists when root is non-empty
+ Files.writeString(dbRoot.resolve(IDENTITY_FILE), "test-id");
+
+ String active = UUID.randomUUID().toString();
+ String inactive = UUID.randomUUID().toString();
+ Files.createDirectories(spaceRoot.resolve(active));
+ Files.createDirectories(spaceRoot.resolve(inactive));
+ Files.writeString(spaceRoot.resolve(ACTIVE_GEN_POINTER), active);
+ // An unrelated file under root should be removed as well
+ Files.writeString(spaceRoot.resolve("leftover.txt"), "x");
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ // After open, only active gen and pointer should remain
+ long dirs = Files.list(spaceRoot).filter(Files::isDirectory).count();
+ long files = Files.list(spaceRoot).filter(Files::isRegularFile).count();
+ assertEquals(dirs, 1);
+ assertEquals(files, 1);
+ assertTrue(Files.exists(spaceRoot.resolve(active)));
+ assertTrue(Files.exists(spaceRoot.resolve(ACTIVE_GEN_POINTER)));
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceFreshBootstrapTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceFreshBootstrapTest.java
new file mode 100644
index 000000000..73232fd2c
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceFreshBootstrapTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVEngine.IDENTITY_FILE;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceFreshBootstrapTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_bootstrap_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testFreshBootstrapCreatesPointerAndGenDir() throws Exception {
+ String spaceId = "space_bootstrap";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot.resolve(spaceId));
+ Files.createDirectories(cpRoot);
+ // Ensure engine identity exists when root is non-empty
+ Files.writeString(dbRoot.resolve(IDENTITY_FILE), "test-id");
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR,
+ Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ Path pointer = dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER);
+ assertTrue(Files.exists(pointer));
+ String uuid = Files.readString(pointer).trim();
+ assertFalse(uuid.isEmpty());
+ assertTrue(Files.isDirectory(dbRoot.resolve(spaceId).resolve(uuid)));
+
+ ICPableKVSpace space = engine.spaces().get(spaceId);
+ space.toWriter().put(ByteString.copyFromUtf8("k"), ByteString.copyFromUtf8("v")).done();
+ try (IKVSpaceReader reader = space.reader()) {
+ assertTrue(reader.get(ByteString.copyFromUtf8("k")).isPresent());
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceLegacyMigrationTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceLegacyMigrationTest.java
new file mode 100644
index 000000000..c61818c8f
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceLegacyMigrationTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVEngine.IDENTITY_FILE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceLegacyMigrationTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ // Prepare a temp root for db and checkpoint
+ tmpRoot = Files.createTempDirectory("kvspace_legacy_migration_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ // Cleanup temp dir recursively
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testLegacyLayoutMigratedInConstructor() throws Exception {
+ String spaceId = "space1";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+ // Ensure engine identity exists when root is non-empty
+ Files.writeString(dbRoot.resolve(IDENTITY_FILE), "test-id");
+
+ // Simulate legacy layout: create a valid RocksDB directly under <dbRoot>/<spaceId>
+ Path legacyRoot = dbRoot.resolve(spaceId);
+ Files.createDirectories(legacyRoot);
+ try (Options options = new Options().setCreateIfMissing(true)) {
+ try (RocksDB db = RocksDB.open(options, legacyRoot.toString())) {
+ // put a tiny entry to ensure DB files created
+ db.put("k".getBytes(), "v".getBytes());
+ }
+ }
+
+ // Build engine and start: constructor of space should migrate legacy layout immediately
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ Map<String, ? extends ICPableKVSpace> spaces = engine.spaces();
+ ICPableKVSpace space = spaces.get(spaceId);
+ assertNotNull(space);
+
+ // Verify pointer file created and points to a uuid directory
+ Path pointer = legacyRoot.resolve(ACTIVE_GEN_POINTER);
+ assertTrue(Files.exists(pointer));
+ String uuid = Files.readString(pointer).trim();
+ assertFalse(uuid.isEmpty());
+ Path currentGenDir = legacyRoot.resolve(uuid);
+ assertTrue(Files.isDirectory(currentGenDir));
+
+ // Write and read to ensure DB works after migration
+ var writer = space.toWriter();
+ writer.put(ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b"));
+ writer.done();
+ try (IKVSpaceReader reader = space.reader()) {
+ var val = reader.get(ByteString.copyFromUtf8("a"));
+ assertTrue(val.isPresent());
+ assertEquals(val.get().toStringUtf8(), "b");
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpacePointerRecoveryTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpacePointerRecoveryTest.java
new file mode 100644
index 000000000..91591cec7
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpacePointerRecoveryTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVEngine.IDENTITY_FILE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpacePointerRecoveryTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_ptr_recover_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testPointerPointsToMissingDir() throws Exception {
+ String spaceId = "space_ptr_missing";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot.resolve(spaceId));
+ Files.createDirectories(cpRoot);
+ // Ensure engine identity exists when root is non-empty
+ Files.writeString(dbRoot.resolve(IDENTITY_FILE), "test-id");
+
+ // Write pointer to a non-existing uuid
+ String missingUUID = UUID.randomUUID().toString();
+ Files.writeString(dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER), missingUUID);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.spaces().get(spaceId);
+ assertNotNull(space);
+
+ String current = Files.readString(dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER)).trim();
+ assertFalse(current.isEmpty());
+ assertNotEquals(missingUUID, current);
+ assertTrue(Files.isDirectory(dbRoot.resolve(spaceId).resolve(current)));
+
+ // simple rw
+ space.toWriter().put(ByteString.copyFromUtf8("k"), ByteString.copyFromUtf8("v")).done();
+ try (IKVSpaceReader reader = space.reader()) {
+ assertEquals(reader.get(ByteString.copyFromUtf8("k")).get().toStringUtf8(), "v");
+ }
+ }
+
+ @Test
+ public void testPointerValidRemains() throws Exception {
+ String spaceId = "space_ptr_valid";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+ Path spaceRoot = dbRoot.resolve(spaceId);
+ Files.createDirectories(spaceRoot);
+ // Ensure engine identity exists when root is non-empty
+ Files.writeString(dbRoot.resolve(IDENTITY_FILE), "test-id");
+ String uuid = UUID.randomUUID().toString();
+ Files.createDirectories(spaceRoot.resolve(uuid));
+ Files.writeString(spaceRoot.resolve(ACTIVE_GEN_POINTER), uuid);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ String after = Files.readString(spaceRoot.resolve(ACTIVE_GEN_POINTER)).trim();
+ assertEquals(after, uuid);
+
+ ICPableKVSpace space = engine.spaces().get(spaceId);
+ space.toWriter().put(ByteString.copyFromUtf8("a"), ByteString.copyFromUtf8("b")).done();
+ try (IKVSpaceReader reader = space.reader()) {
+ assertEquals(reader.get(ByteString.copyFromUtf8("a")).get().toStringUtf8(), "b");
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreAbortTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreAbortTest.java
new file mode 100644
index 000000000..28eeec2e6
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreAbortTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreAbortTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_abort_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testAbortNoEffectAndPointerUnchanged() throws Exception {
+ String spaceId = "space_abort";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+ Path pointer = dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER);
+ String before = Files.readString(pointer).trim();
+
+ IRestoreSession session = space.startRestore((c, b) -> {});
+ session.put(ByteString.copyFromUtf8("k"), ByteString.copyFromUtf8("v"));
+ session.abort();
+
+ // key not present
+ try (IKVSpaceReader reader = space.reader()) {
+ assertFalse(reader.get(ByteString.copyFromUtf8("k")).isPresent());
+ }
+ // pointer unchanged
+ String after = Files.readString(pointer).trim();
+ assertEquals(after, before);
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreBulkTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreBulkTest.java
new file mode 100644
index 000000000..86d19cec1
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreBulkTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Random;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreBulkTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_bulk_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testBulkRestoreReplace() throws Exception {
+ String spaceId = "space_bulk";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+ IRestoreSession session = space.startRestore((c, b) -> {});
+ int n = 5000;
+ for (int i = 0; i < n; i++) {
+ session.put(ByteString.copyFromUtf8("k" + i), ByteString.copyFromUtf8("v" + i));
+ }
+ session.done();
+
+ // random spot-check 10 keys
+ Random r = new Random(0);
+ try (IKVSpaceReader reader = space.reader()) {
+ for (int i = 0; i < 10; i++) {
+ int idx = r.nextInt(n);
+ assertEquals(reader.get(ByteString.copyFromUtf8("k" + idx)).get().toStringUtf8(), "v" + idx);
+ }
+ }
+ // ensure at least one key exists
+ assertTrue(space.size() > 0);
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreFlushListenerTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreFlushListenerTest.java
new file mode 100644
index 000000000..316a36dd4
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreFlushListenerTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreFlushListenerTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_flush_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testBulkRestoreReportsFlushes() throws Exception {
+ String spaceId = "space_flush_bulk";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+ AtomicInteger callbackCount = new AtomicInteger();
+ AtomicLong totalEntries = new AtomicLong();
+ AtomicLong totalBytes = new AtomicLong();
+ IRestoreSession session = space.startRestore((c, b) -> {
+ // collect flush metrics
+ callbackCount.incrementAndGet();
+ totalEntries.addAndGet(c);
+ totalBytes.addAndGet(b);
+ });
+
+ int n = 6000; // exceed default entry budget to trigger flush
+ for (int i = 0; i < n; i++) {
+ ByteString k = ByteString.copyFromUtf8("k" + i);
+ ByteString v = ByteString.copyFromUtf8("v" + i);
+ session.put(k, v);
+ }
+ session.done();
+
+ assertTrue(callbackCount.get() >= 1, "should flush at least once during bulk restore");
+ assertTrue(totalEntries.get() > 0, "reported entries should be positive");
+ assertTrue(totalBytes.get() > 0, "reported bytes should be positive");
+ }
+
+ @Test
+ public void testOverlayRestoreReportsFlushes() throws Exception {
+ String spaceId = "space_flush_overlay";
+ Path dbRoot = tmpRoot.resolve("data2");
+ Path cpRoot = tmpRoot.resolve("cp2");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+ // pre-write something to ensure active DB is not empty
+ space.toWriter().put(ByteString.copyFromUtf8("pre"), ByteString.copyFromUtf8("p")).done();
+
+ AtomicInteger callbackCount = new AtomicInteger();
+ IRestoreSession session = space.startReceiving((c, b) -> callbackCount.incrementAndGet());
+
+ int n = 4000;
+ for (int i = 0; i < n; i++) {
+ ByteString k = ByteString.copyFromUtf8("ok" + i);
+ ByteString v = ByteString.copyFromUtf8("ov" + i);
+ session.put(k, v);
+ }
+ session.done();
+
+ assertTrue(callbackCount.get() >= 1, "should flush at least once during overlay restore");
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreOverlayTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreOverlayTest.java
new file mode 100644
index 000000000..1f2ba4c42
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreOverlayTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreOverlayTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_overlay_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testOverlayKeepsExistingAndOverrides() throws Exception {
+ String spaceId = "space_overlay";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+
+ // pre-existing data
+ ByteString keyA = ByteString.copyFromUtf8("A");
+ ByteString oldA = ByteString.copyFromUtf8("oldA");
+ ByteString keyB = ByteString.copyFromUtf8("B");
+ ByteString oldB = ByteString.copyFromUtf8("oldB");
+ ByteString mx = ByteString.copyFromUtf8("x");
+ ByteString mvx = ByteString.copyFromUtf8("mx");
+ space.toWriter().put(keyA, oldA).put(keyB, oldB).metadata(mx, mvx).done();
+
+ Path pointer = dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER);
+ String before = Files.readString(pointer).trim();
+
+ IRestoreSession session = space.startReceiving((c, b) -> {});
+ ByteString newA = ByteString.copyFromUtf8("newA");
+ ByteString keyC = ByteString.copyFromUtf8("C");
+ ByteString valC = ByteString.copyFromUtf8("valC");
+ ByteString my = ByteString.copyFromUtf8("y");
+ ByteString mvy = ByteString.copyFromUtf8("my");
+ session.put(keyA, newA).put(keyC, valC).metadata(my, mvy);
+ session.done();
+
+ // A overridden, B kept, C added
+ try (IKVSpaceReader reader = space.reader()) {
+ assertEquals(reader.get(keyA).get().toStringUtf8(), "newA");
+ assertEquals(reader.get(keyB).get().toStringUtf8(), "oldB");
+ assertEquals(reader.get(keyC).get().toStringUtf8(), "valC");
+ assertTrue(reader.metadata(mx).isPresent());
+ assertTrue(reader.metadata(my).isPresent());
+ }
+
+ String after = Files.readString(pointer).trim();
+ assertNotEquals(after, before);
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreReplaceTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreReplaceTest.java
new file mode 100644
index 000000000..118f836da
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreReplaceTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreReplaceTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_replace_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testReplaceReplacesAllAndUpdatesPointer() throws Exception {
+ String spaceId = "space_replace";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+ // prepare existing data
+ ByteString k1 = ByteString.copyFromUtf8("k1");
+ ByteString v1 = ByteString.copyFromUtf8("v1");
+ ByteString m1 = ByteString.copyFromUtf8("m1");
+ ByteString mv1 = ByteString.copyFromUtf8("mv1");
+ space.toWriter().put(k1, v1).metadata(m1, mv1).done();
+ try (IKVSpaceReader reader = space.reader()) {
+ assertTrue(reader.exist(k1));
+ }
+
+ Path pointer = dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER);
+ String before = Files.readString(pointer).trim();
+
+ // restore with replace
+ IRestoreSession session = space.startRestore((c, b) -> {});
+ ByteString k2 = ByteString.copyFromUtf8("k2");
+ ByteString v2 = ByteString.copyFromUtf8("v2");
+ ByteString m2 = ByteString.copyFromUtf8("m2");
+ ByteString mv2 = ByteString.copyFromUtf8("mv2");
+ session.put(k2, v2);
+ session.metadata(m2, mv2);
+ session.done();
+
+ // old key removed, new key present
+ try (IKVSpaceReader reader = space.reader()) {
+ assertFalse(reader.exist(k1));
+ assertTrue(reader.exist(k2));
+ assertTrue(reader.metadata(m2).isPresent());
+ assertFalse(reader.metadata(m1).isPresent());
+ }
+
+ String after = Files.readString(pointer).trim();
+ assertNotEquals(after, before);
+
+ // idempotent overwrite in replace mode
+ IRestoreSession s2 = space.startRestore((c, b) -> {});
+ s2.put(k2, ByteString.copyFromUtf8("v3"));
+ s2.put(k2, ByteString.copyFromUtf8("v4"));
+ s2.done();
+ try (IKVSpaceReader reader = space.reader()) {
+ assertEquals(reader.get(k2).get().toStringUtf8(), "v4");
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreRestartCleanupTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreRestartCleanupTest.java
new file mode 100644
index 000000000..c125bb215
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreRestartCleanupTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreRestartCleanupTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_restart_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testRestartCleansInactiveGenerations() throws Exception {
+ String spaceId = "space_restart_cleanup";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+
+ // initial write
+ space.toWriter().put(ByteString.copyFromUtf8("k"), ByteString.copyFromUtf8("v")).done();
+ Path spaceRoot = dbRoot.resolve(spaceId);
+ String before = Files.readString(spaceRoot.resolve(ACTIVE_GEN_POINTER)).trim();
+
+ // restore replace to switch generation
+ IRestoreSession session = space.startRestore((c, b) -> {});
+ session.put(ByteString.copyFromUtf8("k2"), ByteString.copyFromUtf8("v2"));
+ session.done();
+ String after = Files.readString(spaceRoot.resolve(ACTIVE_GEN_POINTER)).trim();
+ assertNotEquals(after, before);
+
+ // stop/start to trigger cleanInactiveOnStartup
+ engine.stop();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ long dirCount;
+ long fileCount;
+ try (Stream<Path> s = Files.list(spaceRoot)) {
+ dirCount = s.filter(Files::isDirectory).count();
+ }
+ try (Stream<Path> s = Files.list(spaceRoot)) {
+ fileCount = s.filter(Files::isRegularFile).count();
+ }
+ assertEquals(dirCount, 1);
+ assertEquals(fileCount, 1);
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreSessionLifecycleTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreSessionLifecycleTest.java
new file mode 100644
index 000000000..0b7d9b1ca
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreSessionLifecycleTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreSessionLifecycleTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_lifecycle_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testSessionClosedAfterDoneOrAbort() throws Exception {
+ String spaceId = "space_lifecycle";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+
+ // done closes session
+ IRestoreSession s1 = space.startRestore((c, b) -> {});
+ s1.put(ByteString.copyFromUtf8("k"), ByteString.copyFromUtf8("v"));
+ s1.done();
+ try (IKVSpaceReader reader = space.reader()) {
+ assertTrue(reader.get(ByteString.copyFromUtf8("k")).isPresent());
+ }
+ assertThrows(IllegalStateException.class, () -> s1.put(ByteString.copyFromUtf8("k2"), ByteString.copyFromUtf8("v2")));
+ assertThrows(IllegalStateException.class, () -> s1.metadata(ByteString.copyFromUtf8("m"), ByteString.copyFromUtf8("mv")));
+
+ // abort closes session
+ IRestoreSession s2 = space.startRestore((c, b) -> {});
+ s2.put(ByteString.copyFromUtf8("x"), ByteString.copyFromUtf8("y"));
+ s2.abort();
+ assertThrows(IllegalStateException.class, () -> s2.put(ByteString.copyFromUtf8("x2"), ByteString.copyFromUtf8("y2")));
+ assertThrows(IllegalStateException.class, () -> s2.metadata(ByteString.copyFromUtf8("m2"), ByteString.copyFromUtf8("mv2")));
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreSwitchPointerTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreSwitchPointerTest.java
new file mode 100644
index 000000000..339ff8817
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRestoreSwitchPointerTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBKVEngine.IDENTITY_FILE;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IKVSpaceRefreshableReader;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRestoreSwitchPointerTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_restore_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testPointerUpdatedAfterRestoreReplace() throws Exception {
+ String spaceId = "space_restore";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot.resolve(spaceId));
+ Files.createDirectories(cpRoot);
+ // Ensure engine identity exists when root is non-empty
+ Files.writeString(dbRoot.resolve(IDENTITY_FILE), "test-id");
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.spaces().get(spaceId);
+
+ Path pointer = dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER);
+ String before = Files.readString(pointer).trim();
+
+ IRestoreSession session = space.startRestore((c, b) -> {});
+ session.put(ByteString.copyFromUtf8("k1"), ByteString.copyFromUtf8("v1"));
+ session.done();
+
+ String after = Files.readString(pointer).trim();
+ assertNotEquals(after, before);
+ try (IKVSpaceRefreshableReader reader = space.reader()) {
+ assertTrue(reader.get(ByteString.copyFromUtf8("k1")).isPresent());
+ }
+ }
+}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRuntimeCleanerTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRuntimeCleanerTest.java
new file mode 100644
index 000000000..06d6a2602
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBCPableKVSpaceRuntimeCleanerTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.awaitility.Awaitility.await;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.stream.Stream;
+import org.apache.bifromq.basekv.localengine.ICPableKVSpace;
+import org.apache.bifromq.basekv.localengine.IRestoreSession;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBCPableKVSpaceRuntimeCleanerTest {
+ private Path tmpRoot;
+ private RocksDBCPableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_runtime_cleaner_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testCleanerDeletesInactiveGenerationAtRuntime() throws Exception {
+ String spaceId = "space_runtime_cleaner";
+ Path dbRoot = tmpRoot.resolve("data");
+ Path cpRoot = tmpRoot.resolve("cp");
+ Files.createDirectories(dbRoot);
+ Files.createDirectories(cpRoot);
+
+ Struct conf = RocksDBDefaultConfigs.CP.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .putFields(RocksDBDefaultConfigs.DB_CHECKPOINT_ROOT_DIR, Value.newBuilder().setStringValue(cpRoot.toString()).build())
+ .build();
+ engine = new RocksDBCPableKVEngine(null, conf);
+ engine.start("tag", "value");
+
+ ICPableKVSpace space = engine.createIfMissing(spaceId);
+ // write something to current generation
+ space.toWriter().put(ByteString.copyFromUtf8("k"), ByteString.copyFromUtf8("v")).done();
+
+ Path spaceRoot = dbRoot.resolve(spaceId);
+ Path pointer = spaceRoot.resolve(ACTIVE_GEN_POINTER);
+ String before = Files.readString(pointer).trim();
+ assertTrue(Files.exists(spaceRoot.resolve(before)));
+
+ // Replace restore to force generation switch
+ IRestoreSession session = space.startRestore((c, b) -> {});
+ session.put(ByteString.copyFromUtf8("k2"), ByteString.copyFromUtf8("v2"));
+ session.done();
+
+ String after = Files.readString(pointer).trim();
+ assertNotEquals(after, before);
+ // New generation directory should exist immediately
+ assertTrue(Files.exists(spaceRoot.resolve(after)));
+
+ // Await eventual cleanup by Cleaner/GC to avoid relying on immediate deletion
+ await().atMost(Duration.ofSeconds(10)).pollInterval(Duration.ofMillis(100)).untilAsserted(() -> {
+ // Hint GC to speed up Cleaner, but do not rely on it
+ System.gc();
+ System.runFinalization();
+
+ // Old generation directory should eventually be deleted at runtime by cleaner
+ assertTrue(Files.notExists(spaceRoot.resolve(before)));
+
+ long dirCount;
+ long fileCount;
+ try (Stream<Path> s = Files.list(spaceRoot)) {
+ dirCount = s.filter(Files::isDirectory).count();
+ }
+ try (Stream<Path> s = Files.list(spaceRoot)) {
+ fileCount = s.filter(Files::isRegularFile).count();
+ }
+ assertEquals(dirCount, 1);
+ assertEquals(fileCount, 1);
+ });
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTriggerTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTriggerTest.java
similarity index 100%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTriggerTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTriggerTest.java
index bba276e1e..2cf4cb314 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTriggerTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBKVSpaceCompactionTriggerTest.java
@@ -22,8 +22,8 @@ package org.apache.bifromq.basekv.localengine.rocksdb;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import org.apache.bifromq.basekv.localengine.MockableTest;
import java.util.stream.IntStream;
+import org.apache.bifromq.basekv.localengine.MockableTest;
import org.mockito.Mock;
import org.testng.annotations.Test;
diff --git a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
similarity index 67%
rename from base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
rename to base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
index ca4c46187..5111207c9 100644
--- a/base-kv/base-kv-local-engine/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVEngineTest.java
@@ -19,27 +19,29 @@
package org.apache.bifromq.basekv.localengine.rocksdb;
-import org.apache.bifromq.basekv.localengine.IKVEngine;
-import org.apache.bifromq.basekv.localengine.IKVSpace;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import java.nio.file.Paths;
import lombok.SneakyThrows;
+import org.apache.bifromq.basekv.localengine.IKVEngine;
+import org.apache.bifromq.basekv.localengine.IWALableKVSpace;
-public class RocksDBWALableKVEngineTest extends AbstractRocksDBKVEngineTest {
- protected RocksDBWALableKVEngineConfigurator configurator;
+public class RocksDBWALableKVEngineTest extends AbstractRocksDBWALableEngineTest {
+ protected Struct conf;
@SneakyThrows
@Override
protected void beforeStart() {
super.beforeStart();
String DB_NAME = "testDB";
- configurator = RocksDBWALableKVEngineConfigurator.builder()
- .dbRootDir(Paths.get(dbRootDir.toString(), DB_NAME).toString())
+ conf = RocksDBDefaultConfigs.WAL.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(Paths.get(dbRootDir.toString(), DB_NAME).toString()).build())
.build();
}
@SneakyThrows
@Override
- protected IKVEngine<? extends IKVSpace> newEngine() {
- return new RocksDBWALableKVEngine(null, configurator);
+ protected IKVEngine<? extends IWALableKVSpace> newEngine() {
+ return new RocksDBWALableKVEngine(null, conf);
}
}
diff --git a/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpaceNoPointerTest.java b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpaceNoPointerTest.java
new file mode 100644
index 000000000..400efc3ff
--- /dev/null
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/java/org/apache/bifromq/basekv/localengine/rocksdb/RocksDBWALableKVSpaceNoPointerTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine.rocksdb;
+
+import static org.apache.bifromq.basekv.localengine.rocksdb.RocksDBCPableKVSpace.ACTIVE_GEN_POINTER;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class RocksDBWALableKVSpaceNoPointerTest {
+ private Path tmpRoot;
+ private RocksDBWALableKVEngine engine;
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ tmpRoot = Files.createTempDirectory("kvspace_wal_");
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ try {
+ if (engine != null) {
+ engine.stop();
+ }
+ } catch (Throwable ignore) {
+ }
+ if (tmpRoot != null) {
+ Files.walk(tmpRoot)
+ .sorted((a, b) -> b.getNameCount() - a.getNameCount())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (Throwable ignore) {
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testNoPointerCreatedForWALable() throws Exception {
+ String spaceId = "wal_space";
+ Path dbRoot = tmpRoot.resolve("data");
+ Files.createDirectories(dbRoot);
+
+ Struct conf = RocksDBDefaultConfigs.WAL.toBuilder()
+ .putFields(RocksDBDefaultConfigs.DB_ROOT_DIR, Value.newBuilder().setStringValue(dbRoot.toString()).build())
+ .build();
+ engine = new RocksDBWALableKVEngine(null, conf);
+ engine.start("tag", "value");
+ engine.createIfMissing(spaceId);
+
+ Path pointer = dbRoot.resolve(spaceId).resolve(ACTIVE_GEN_POINTER);
+ assertFalse(Files.exists(pointer));
+ // DB files should be directly under space root
+ assertTrue(Files.list(dbRoot.resolve(spaceId)).anyMatch(Files::isRegularFile));
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/test/resources/log4j2-test.xml b/base-kv/base-kv-local-engine-rocksdb/src/test/resources/log4j2-test.xml
similarity index 99%
copy from base-kv/base-kv-local-engine/src/test/resources/log4j2-test.xml
copy to base-kv/base-kv-local-engine-rocksdb/src/test/resources/log4j2-test.xml
index 7b038d5b4..1404c8668 100644
--- a/base-kv/base-kv-local-engine/src/test/resources/log4j2-test.xml
+++ b/base-kv/base-kv-local-engine-rocksdb/src/test/resources/log4j2-test.xml
@@ -35,3 +35,4 @@
</Root>
</Loggers>
</Configuration>
+
diff --git a/base-kv/base-kv-local-engine/pom.xml b/base-kv/base-kv-local-engine-spi/pom.xml
similarity index 85%
rename from base-kv/base-kv-local-engine/pom.xml
rename to base-kv/base-kv-local-engine-spi/pom.xml
index 398e6036a..4c729591d 100644
--- a/base-kv/base-kv-local-engine/pom.xml
+++ b/base-kv/base-kv-local-engine-spi/pom.xml
@@ -27,10 +27,13 @@
<artifactId>base-kv</artifactId>
<version>${revision}</version>
</parent>
-
- <artifactId>base-kv-local-engine</artifactId>
+ <artifactId>base-kv-local-engine-spi</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.apache.bifromq</groupId>
+ <artifactId>base-hookloader</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.bifromq</groupId>
<artifactId>base-kv-type-proto</artifactId>
@@ -43,6 +46,10 @@
<groupId>org.apache.bifromq</groupId>
<artifactId>base-logger</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.pcollections</groupId>
+ <artifactId>pcollections</artifactId>
+ </dependency>
<dependency>
<groupId>io.reactivex.rxjava3</groupId>
<artifactId>rxjava</artifactId>
@@ -51,10 +58,6 @@
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
</dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- </dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -71,10 +74,11 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
- <!-- for testing-->
+ <!-- test -->
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -114,6 +118,16 @@
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
</plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVEngine.java b/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVEngine.java
similarity index 65%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVEngine.java
rename to base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVEngine.java
index fa6c77b05..b58284fcd 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVEngine.java
+++ b/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVEngine.java
@@ -22,6 +22,8 @@ package org.apache.bifromq.basekv.localengine;
import static com.google.common.collect.Lists.newArrayList;
import com.google.common.collect.Iterables;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
@@ -30,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
+import lombok.NonNull;
import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
import org.apache.bifromq.logger.MDCLogger;
import org.slf4j.Logger;
@@ -38,20 +41,20 @@ import org.slf4j.Logger;
* The abstract class of KVEngine.
*
* @param <T> the type of KV space created by the engine
- * @param <C> the type of configurator
*/
-public abstract class AbstractKVEngine<T extends IKVSpace, C extends IKVEngineConfigurator> implements IKVEngine<T> {
+public abstract class AbstractKVEngine<T extends IKVSpace> implements IKVEngine<T> {
protected final String overrideIdentity;
- protected final C configurator;
+ protected final Struct engineConf;
private final AtomicReference<State> state = new AtomicReference<>(State.INIT);
private final Map<String, T> kvSpaceMap = new ConcurrentHashMap<>();
protected Logger log;
protected String[] metricTags;
private Gauge gauge;
- public AbstractKVEngine(String overrideIdentity, C configurator) {
+ public AbstractKVEngine(String overrideIdentity, @NonNull Struct conf) {
+ validateConf(conf);
this.overrideIdentity = overrideIdentity;
- this.configurator = configurator;
+ this.engineConf = conf;
}
@Override
@@ -111,34 +114,73 @@ public abstract class AbstractKVEngine<T extends IKVSpace, C extends IKVEngineCo
assertStarted();
return kvSpaceMap.computeIfAbsent(spaceId,
k -> {
- T space = buildKVSpace(spaceId, configurator, () -> kvSpaceMap.remove(spaceId), metricTags);
+ T space = buildKVSpace(spaceId, engineConf, () -> kvSpaceMap.remove(spaceId), metricTags);
space.open();
return space;
});
}
protected final void load(String spaceId) {
- T space = buildKVSpace(spaceId, configurator, () -> kvSpaceMap.remove(spaceId), metricTags);
+ T space = buildKVSpace(spaceId, engineConf, () -> kvSpaceMap.remove(spaceId), metricTags);
space.open();
T prev = kvSpaceMap.put(spaceId, space);
assert prev == null;
}
- private T buildKVSpace(String spaceId, C configurator, Runnable onDestroy, String... tags) {
+ private T buildKVSpace(String spaceId, Struct conf, Runnable onDestroy, String... tags) {
String[] tagList =
newArrayList(Iterables.concat(List.of(tags), List.of("spaceId", spaceId))).toArray(String[]::new);
KVSpaceOpMeters opMeters = new KVSpaceOpMeters(spaceId, Tags.of(tagList));
Logger logger = MDCLogger.getLogger("space.logger", tagList);
- return doBuildKVSpace(spaceId, configurator, onDestroy, opMeters, logger, tagList);
+ return doBuildKVSpace(spaceId, conf, onDestroy, opMeters, logger, tagList);
}
protected abstract T doBuildKVSpace(String spaceId,
- C configurator,
+ Struct conf,
Runnable onDestroy,
KVSpaceOpMeters opMeters,
Logger logger,
String... tags);
+ /**
+ * Validate provided configuration is complete and valid.
+ * This method enforces structural completeness based on defaultConf,
+ * non-empty string constraints from nonEmptyStringKeys, then delegates to validateSemantics.
+ *
+ * @param conf caller provided conf
+ * @throws IllegalArgumentException if conf is invalid
+ */
+ protected final void validateConf(Struct conf) {
+ if (conf == null) {
+ throw new IllegalArgumentException("Engine configuration must not be null");
+ }
+ Struct defaults = defaultConf();
+ // Check required keys and non-null values
+ for (Map.Entry<String, Value> e : defaults.getFieldsMap().entrySet()) {
+ String key = e.getKey();
+ if (!conf.getFieldsMap().containsKey(key)) {
+ throw new IllegalArgumentException("Missing required config key: " + key);
+ }
+ Value v = conf.getFieldsMap().get(key);
+ if (v.getKindCase() == Value.KindCase.KIND_NOT_SET || v.hasNullValue()) {
+ throw new IllegalArgumentException("Config key has null value: " + key);
+ }
+ }
+ validateSemantics(conf);
+ }
+
+ /**
+ * Engine default full configuration used for completeness checking.
+ */
+ protected abstract Struct defaultConf();
+
+ /**
+ * Additional engine-specific semantic validation.
+ */
+ protected void validateSemantics(Struct conf) {
+ // no-op by default
+ }
+
private enum State {
INIT, STARTING, STARTED, FATAL_FAILURE, STOPPING, STOPPED
}
diff --git a/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpace.java b/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpace.java
new file mode 100644
index 000000000..4a366deba
--- /dev/null
+++ b/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpace.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bifromq.basekv.localengine;
+
+import static io.reactivex.rxjava3.subjects.BehaviorSubject.createDefault;
+import static java.util.Collections.emptyMap;
+
+import com.google.protobuf.ByteString;
+import io.micrometer.core.instrument.Tags;
+import io.reactivex.rxjava3.core.Observable;
+import io.reactivex.rxjava3.subjects.BehaviorSubject;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
+import org.slf4j.Logger;
+
+/**
+ * Base implementation of IKVSpace.
+ *
+ * @param <P> the type of epoch
+ */
+public abstract class AbstractKVSpace<P extends IKVSpaceEpoch> implements IKVSpace {
+ protected final String id;
+ protected final KVSpaceOpMeters opMeters;
+ protected final Logger logger;
+ protected final Tags tags;
+ private final AtomicReference<State> state;
+ private final Runnable onDestroy;
+ private final BehaviorSubject<Map<ByteString, ByteString>> metadataSubject;
+
+ public AbstractKVSpace(String id,
+ Runnable onDestroy,
+ KVSpaceOpMeters opMeters,
+ Logger logger,
+ String... tags) {
+ this.id = id;
+ this.opMeters = opMeters;
+ this.logger = logger;
+ this.onDestroy = onDestroy;
+ state = new AtomicReference<>(State.Init);
+ metadataSubject = createDefault(emptyMap());
+ this.tags = Tags.of(tags).and("spaceId", id);
+ }
+
+ @Override
+ public final String id() {
+ return id;
+ }
+
+ @Override
+ public final void open() {
+ if (state.compareAndSet(State.Init, State.Opening)) {
+ doOpen();
+ }
+ }
+
+ @Override
+ public final Observable<Map<ByteString, ByteString>> metadata() {
+ return metadataSubject;
+ }
+
+ @Override
+ public final KVSpaceDescriptor describe() {
+ return new KVSpaceDescriptor(id, collectStats());
+ }
+
+ @Override
+ public final long size() {
+ return opMeters.sizeCallTimer.record(() -> doSize(Boundary.getDefaultInstance()));
+ }
+
+ @Override
+ public final void destroy() {
+ close();
+ if (state.compareAndSet(State.Closed, State.Destroying)) {
+ try {
+ doDestroy();
+ } catch (Throwable e) {
+ throw new KVEngineException("Destroy KVRange error", e);
+ } finally {
+ onDestroy.run();
+ state.set(State.Terminated);
+ }
+ }
+ }
+
+ @Override
+ public final void close() {
+ if (state.compareAndSet(State.Opening, State.Closing)) {
+ try {
+ doClose();
+ metadataSubject.onComplete();
+ } finally {
+ state.set(State.Closed);
+ }
+ }
+ }
+
+ protected Map<ByteString, ByteString> currentMetadata() {
+ return metadataSubject.getValue();
+ }
+
+ protected void updateMetadata(Map<ByteString, ByteString> newMetadata) {
+ metadataSubject.onNext(newMetadata);
+ }
+
+ protected final State state() {
+ return state.get();
+ }
+
+ protected abstract void doClose();
+
+ protected void doDestroy() {
+ }
+
+ protected abstract P handle();
+
+ protected abstract void doOpen();
+
+ protected abstract long doSize(Boundary boundary);
+
+ private Map<String, Double> collectStats() {
+ Map<String, Double> stats = new HashMap<>();
+ stats.put("size", (double) size());
+ // TODO: more stats
+ return stats;
+ }
+
+ protected enum State {
+ Init, Opening, Destroying, Closing, Closed, Terminated
+ }
+}
diff --git a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpaceReader.java b/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpaceReader.java
similarity index 89%
rename from base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpaceReader.java
rename to base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpaceReader.java
index e176eb21a..f2b6df9ff 100644
--- a/base-kv/base-kv-local-engine/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpaceReader.java
+++ b/base-kv/base-kv-local-engine-spi/src/main/java/org/apache/bifromq/basekv/localengine/AbstractKVSpaceReader.java
@@ -19,12 +19,15 @@
package org.apache.bifromq.basekv.localengine;
-import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
-import org.apache.bifromq.basekv.proto.Boundary;
import com.google.protobuf.ByteString;
import java.util.Optional;
+import org.apache.bifromq.basekv.localengine.metrics.KVSpaceOpMeters;
+import org.apache.bifromq.basekv.proto.Boundary;
import org.slf4j.Logger;
+/**
+ * The base implementation of IKVSpaceReader with operation metrics.
+ */
public abstract class AbstractKVSpaceReader implements IKVSpaceReader {
protected final String id;
protected final KVSpaceOpMeters opMeters;
@@ -46,27 +49,11 @@ public abstract class AbstractKVSpaceReader implements IKVSpaceReader {
return opMeters.metadataCallTimer.record(() -> doMetadata(metaKey));
}
- protected abstract Optional<ByteString> doMetadata(ByteString metaKey);
-
- @Override
- public final long size() {
- return size(Boundary.getDefaultInstance());
- }
-
- @Override
- public final long size(Boundary boundary) {
- return opMeters.sizeCallTimer.record(() -> doSize(boundary));
- }
-
- protected abstract long doSize(Boundary boundary);
-
@Override
public final boolean exist(ByteString key) {
return opMeters.existCallTimer.record(() -> doExist(key));
}
- protected abstract boolean doExist(ByteString key);
-
... 31132 lines suppressed ...