You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ab...@apache.org on 2023/12/19 21:10:30 UTC
(kafka) branch trunk updated: KAFKA-14412: Generalise over RocksDB WriteBatch (#14853)
This is an automated email from the ASF dual-hosted git repository.
ableegoldman pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new b903d786ba8 KAFKA-14412: Generalise over RocksDB WriteBatch (#14853)
b903d786ba8 is described below
commit b903d786ba8b5a7a29d0c41796a9846104bee296
Author: Nick Telford <ni...@gmail.com>
AuthorDate: Tue Dec 19 21:10:24 2023 +0000
KAFKA-14412: Generalise over RocksDB WriteBatch (#14853)
* KAFKA-14412: Generalise over RocksDB WriteBatch
The type hierarchy of RocksDB's `WriteBatch` looks like this:
```
+---------------------+
| WriteBatchInterface |
+---------------------+
^
|
+---------------------+
| AbstractWriteBatch |
+---------------------+
^
|
+----------+----------+
| |
+------------+ +---------------------+
| WriteBatch | | WriteBatchWithIndex |
+------------+ +---------------------+
```
By switching our `BatchWritingStore` methods from `WriteBatch` to
`WriteBatchInterface`, we enable the use of `WriteBatchWithIndex` as
well.
* Improve error reporting for unknown batch type
We should be using an `IllegalStateException`, and we should log a
message informing the user that this is a bug.
This branch should be unreachable, as both of the possible
implementations of `WriteBatchInterface` are matched in the previous two
branches.
Reviewers: Anna Sophie Blee-Goldman <ab...@apache.org>
---
.../streams/state/internals/BatchWritingStore.java | 6 +++---
.../state/internals/LogicalKeyValueSegment.java | 6 +++---
.../streams/state/internals/RocksDBStore.java | 25 ++++++++++++++++------
.../state/internals/RocksDBTimestampedStore.java | 6 +++---
4 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java
index 2ac1e3bc39e..ef89e127178 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java
@@ -18,10 +18,10 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.streams.KeyValue;
import org.rocksdb.RocksDBException;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
public interface BatchWritingStore {
void addToBatch(final KeyValue<byte[], byte[]> record,
- final WriteBatch batch) throws RocksDBException;
- void write(final WriteBatch batch) throws RocksDBException;
+ final WriteBatchInterface batch) throws RocksDBException;
+ void write(final WriteBatchInterface batch) throws RocksDBException;
}
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java
index 0c49749e8ac..4de5c78a7b8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java
@@ -38,7 +38,7 @@ import org.apache.kafka.streams.state.internals.RocksDBVersionedStore.VersionedS
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDBException;
import org.rocksdb.Snapshot;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -239,7 +239,7 @@ class LogicalKeyValueSegment implements Comparable<LogicalKeyValueSegment>, Segm
}
@Override
- public void addToBatch(final KeyValue<byte[], byte[]> record, final WriteBatch batch) throws RocksDBException {
+ public void addToBatch(final KeyValue<byte[], byte[]> record, final WriteBatchInterface batch) throws RocksDBException {
physicalStore.addToBatch(
new KeyValue<>(
prefixKeyFormatter.addPrefix(record.key),
@@ -248,7 +248,7 @@ class LogicalKeyValueSegment implements Comparable<LogicalKeyValueSegment>, Segm
}
@Override
- public void write(final WriteBatch batch) throws RocksDBException {
+ public void write(final WriteBatchInterface batch) throws RocksDBException {
// no key transformations here since they should've already been done as part
// of adding to the write batch
physicalStore.write(batch);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 86b7b1d34b1..90526b6fa97 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -61,6 +61,8 @@ import org.rocksdb.Snapshot;
import org.rocksdb.Statistics;
import org.rocksdb.TableFormatConfig;
import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
+import org.rocksdb.WriteBatchWithIndex;
import org.rocksdb.WriteOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -652,13 +654,22 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
@Override
public void addToBatch(final KeyValue<byte[], byte[]> record,
- final WriteBatch batch) throws RocksDBException {
+ final WriteBatchInterface batch) throws RocksDBException {
dbAccessor.addToBatch(record.key, record.value, batch);
}
@Override
- public void write(final WriteBatch batch) throws RocksDBException {
- db.write(wOptions, batch);
+ public void write(final WriteBatchInterface batch) throws RocksDBException {
+ if (batch instanceof WriteBatch) {
+ db.write(wOptions, (WriteBatch) batch);
+ } else if (batch instanceof WriteBatchWithIndex) {
+ db.write(wOptions, (WriteBatchWithIndex) batch);
+ } else {
+ log.error("Unknown type of batch {}. This is a bug in Kafka Streams. " +
+ "Please file a bug report at https://issues.apache.org/jira/projects/KAFKA.",
+ batch.getClass().getCanonicalName());
+ throw new IllegalStateException("Unknown type of batch " + batch.getClass().getCanonicalName());
+ }
}
@Override
@@ -719,7 +730,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
final byte[] value);
void prepareBatch(final List<KeyValue<Bytes, byte[]>> entries,
- final WriteBatch batch) throws RocksDBException;
+ final WriteBatchInterface batch) throws RocksDBException;
byte[] get(final byte[] key) throws RocksDBException;
@@ -752,7 +763,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
void addToBatch(final byte[] key,
final byte[] value,
- final WriteBatch batch) throws RocksDBException;
+ final WriteBatchInterface batch) throws RocksDBException;
void close();
}
@@ -786,7 +797,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
@Override
public void prepareBatch(final List<KeyValue<Bytes, byte[]>> entries,
- final WriteBatch batch) throws RocksDBException {
+ final WriteBatchInterface batch) throws RocksDBException {
for (final KeyValue<Bytes, byte[]> entry : entries) {
Objects.requireNonNull(entry.key, "key cannot be null");
addToBatch(entry.key.get(), entry.value, batch);
@@ -869,7 +880,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
@Override
public void addToBatch(final byte[] key,
final byte[] value,
- final WriteBatch batch) throws RocksDBException {
+ final WriteBatchInterface batch) throws RocksDBException {
if (value == null) {
batch.delete(columnFamily, key);
} else {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
index 6f54c385f13..1b8884cd330 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
@@ -32,7 +32,7 @@ import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -132,7 +132,7 @@ public class RocksDBTimestampedStore extends RocksDBStore implements Timestamped
@Override
public void prepareBatch(final List<KeyValue<Bytes, byte[]>> entries,
- final WriteBatch batch) throws RocksDBException {
+ final WriteBatchInterface batch) throws RocksDBException {
for (final KeyValue<Bytes, byte[]> entry : entries) {
Objects.requireNonNull(entry.key, "key cannot be null");
addToBatch(entry.key.get(), entry.value, batch);
@@ -256,7 +256,7 @@ public class RocksDBTimestampedStore extends RocksDBStore implements Timestamped
@Override
public void addToBatch(final byte[] key,
final byte[] value,
- final WriteBatch batch) throws RocksDBException {
+ final WriteBatchInterface batch) throws RocksDBException {
if (value == null) {
batch.delete(oldColumnFamily, key);
batch.delete(newColumnFamily, key);