You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ab...@apache.org on 2023/12/19 21:10:30 UTC

(kafka) branch trunk updated: KAFKA-14412: Generalise over RocksDB WriteBatch (#14853)

This is an automated email from the ASF dual-hosted git repository.

ableegoldman pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new b903d786ba8 KAFKA-14412: Generalise over RocksDB WriteBatch (#14853)
b903d786ba8 is described below

commit b903d786ba8b5a7a29d0c41796a9846104bee296
Author: Nick Telford <ni...@gmail.com>
AuthorDate: Tue Dec 19 21:10:24 2023 +0000

    KAFKA-14412: Generalise over RocksDB WriteBatch (#14853)
    
    * KAFKA-14412: Generalise over RocksDB WriteBatch
    
    The type hierarchy of RocksDB's `WriteBatch` looks like this:
    
    ```
            +---------------------+
            | WriteBatchInterface |
            +---------------------+
                       ^
                       |
            +---------------------+
            |  AbstractWriteBatch |
            +---------------------+
                       ^
                       |
            +----------+----------+
            |                     |
     +------------+    +---------------------+
     | WriteBatch |    | WriteBatchWithIndex |
     +------------+    +---------------------+
    ```
    
    By switching our `BatchWritingStore` methods from `WriteBatch` to
    `WriteBatchInterface`, we enable the use of `WriteBatchWithIndex` as
    well.
    
    * Improve error reporting for unknown batch type
    
    We should be using an `IllegalStateException`, and we should log a
    message informing the user that this is a bug.
    
    This branch should be unreachable, as both of the possible
    implementations of `WriteBatchInterface` are matched in the previous two
    branches.
    
    Reviewers: Anna Sophie Blee-Goldman <ab...@apache.org>
---
 .../streams/state/internals/BatchWritingStore.java |  6 +++---
 .../state/internals/LogicalKeyValueSegment.java    |  6 +++---
 .../streams/state/internals/RocksDBStore.java      | 25 ++++++++++++++++------
 .../state/internals/RocksDBTimestampedStore.java   |  6 +++---
 4 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java
index 2ac1e3bc39e..ef89e127178 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/BatchWritingStore.java
@@ -18,10 +18,10 @@ package org.apache.kafka.streams.state.internals;
 
 import org.apache.kafka.streams.KeyValue;
 import org.rocksdb.RocksDBException;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
 
 public interface BatchWritingStore {
     void addToBatch(final KeyValue<byte[], byte[]> record,
-                    final WriteBatch batch) throws RocksDBException;
-    void write(final WriteBatch batch) throws RocksDBException;
+                    final WriteBatchInterface batch) throws RocksDBException;
+    void write(final WriteBatchInterface batch) throws RocksDBException;
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java
index 0c49749e8ac..4de5c78a7b8 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/LogicalKeyValueSegment.java
@@ -38,7 +38,7 @@ import org.apache.kafka.streams.state.internals.RocksDBVersionedStore.VersionedS
 import org.rocksdb.ReadOptions;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.Snapshot;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -239,7 +239,7 @@ class LogicalKeyValueSegment implements Comparable<LogicalKeyValueSegment>, Segm
     }
 
     @Override
-    public void addToBatch(final KeyValue<byte[], byte[]> record, final WriteBatch batch) throws RocksDBException {
+    public void addToBatch(final KeyValue<byte[], byte[]> record, final WriteBatchInterface batch) throws RocksDBException {
         physicalStore.addToBatch(
             new KeyValue<>(
                 prefixKeyFormatter.addPrefix(record.key),
@@ -248,7 +248,7 @@ class LogicalKeyValueSegment implements Comparable<LogicalKeyValueSegment>, Segm
     }
 
     @Override
-    public void write(final WriteBatch batch) throws RocksDBException {
+    public void write(final WriteBatchInterface batch) throws RocksDBException {
         // no key transformations here since they should've already been done as part
         // of adding to the write batch
         physicalStore.write(batch);
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 86b7b1d34b1..90526b6fa97 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -61,6 +61,8 @@ import org.rocksdb.Snapshot;
 import org.rocksdb.Statistics;
 import org.rocksdb.TableFormatConfig;
 import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
+import org.rocksdb.WriteBatchWithIndex;
 import org.rocksdb.WriteOptions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -652,13 +654,22 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
 
     @Override
     public void addToBatch(final KeyValue<byte[], byte[]> record,
-                           final WriteBatch batch) throws RocksDBException {
+                           final WriteBatchInterface batch) throws RocksDBException {
         dbAccessor.addToBatch(record.key, record.value, batch);
     }
 
     @Override
-    public void write(final WriteBatch batch) throws RocksDBException {
-        db.write(wOptions, batch);
+    public void write(final WriteBatchInterface batch) throws RocksDBException {
+        if (batch instanceof WriteBatch) {
+            db.write(wOptions, (WriteBatch) batch);
+        } else if (batch instanceof WriteBatchWithIndex) {
+            db.write(wOptions, (WriteBatchWithIndex) batch);
+        } else {
+            log.error("Unknown type of batch {}. This is a bug in Kafka Streams. " +
+                    "Please file a bug report at https://issues.apache.org/jira/projects/KAFKA.",
+                    batch.getClass().getCanonicalName());
+            throw new IllegalStateException("Unknown type of batch " + batch.getClass().getCanonicalName());
+        }
     }
 
     @Override
@@ -719,7 +730,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
                  final byte[] value);
 
         void prepareBatch(final List<KeyValue<Bytes, byte[]>> entries,
-                          final WriteBatch batch) throws RocksDBException;
+                          final WriteBatchInterface batch) throws RocksDBException;
 
         byte[] get(final byte[] key) throws RocksDBException;
 
@@ -752,7 +763,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
 
         void addToBatch(final byte[] key,
                         final byte[] value,
-                        final WriteBatch batch) throws RocksDBException;
+                        final WriteBatchInterface batch) throws RocksDBException;
 
         void close();
     }
@@ -786,7 +797,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
 
         @Override
         public void prepareBatch(final List<KeyValue<Bytes, byte[]>> entries,
-                                 final WriteBatch batch) throws RocksDBException {
+                                 final WriteBatchInterface batch) throws RocksDBException {
             for (final KeyValue<Bytes, byte[]> entry : entries) {
                 Objects.requireNonNull(entry.key, "key cannot be null");
                 addToBatch(entry.key.get(), entry.value, batch);
@@ -869,7 +880,7 @@ public class RocksDBStore implements KeyValueStore<Bytes, byte[]>, BatchWritingS
         @Override
         public void addToBatch(final byte[] key,
                                final byte[] value,
-                               final WriteBatch batch) throws RocksDBException {
+                               final WriteBatchInterface batch) throws RocksDBException {
             if (value == null) {
                 batch.delete(columnFamily, key);
             } else {
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
index 6f54c385f13..1b8884cd330 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBTimestampedStore.java
@@ -32,7 +32,7 @@ import org.rocksdb.ReadOptions;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.RocksIterator;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteBatchInterface;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -132,7 +132,7 @@ public class RocksDBTimestampedStore extends RocksDBStore implements Timestamped
 
         @Override
         public void prepareBatch(final List<KeyValue<Bytes, byte[]>> entries,
-                                 final WriteBatch batch) throws RocksDBException {
+                                 final WriteBatchInterface batch) throws RocksDBException {
             for (final KeyValue<Bytes, byte[]> entry : entries) {
                 Objects.requireNonNull(entry.key, "key cannot be null");
                 addToBatch(entry.key.get(), entry.value, batch);
@@ -256,7 +256,7 @@ public class RocksDBTimestampedStore extends RocksDBStore implements Timestamped
         @Override
         public void addToBatch(final byte[] key,
                                final byte[] value,
-                               final WriteBatch batch) throws RocksDBException {
+                               final WriteBatchInterface batch) throws RocksDBException {
             if (value == null) {
                 batch.delete(oldColumnFamily, key);
                 batch.delete(newColumnFamily, key);