You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by lu...@apache.org on 2023/12/22 15:36:23 UTC

(kafka) 02/02: KAFKA-16046: fix stream-stream-join store types (#15061)

This is an automated email from the ASF dual-hosted git repository.

lucasbru pushed a commit to branch 3.7
in repository https://gitbox.apache.org/repos/asf/kafka.git

commit 2488fb71b95e3aa58d584058f9eccff6f4297bc7
Author: Almog Gavra <al...@gmail.com>
AuthorDate: Fri Dec 22 06:23:58 2023 -0800

    KAFKA-16046: fix stream-stream-join store types (#15061)
    
    Before #14648, the KStreamImplJoin class would always create non-timestamped persistent windowed stores. After that PR, the default was changed to create timestamped stores. This wasn't compatible because, during restoration, timestamped stores have their values transformed to prepend the timestamp to the value. This caused serialization errors when trying to read from the store because the deserializers did not expect the timestamp to be prepended.
    
    To fix this, we allow creating non-timestamped stores using the DslWindowParams
    
    Testing was done both manually as well as adding a unit test to ensure that the stores created are not timestamped. I also confirmed that the only place in the code persistentWindowStore was used before #14648 was in the StreamJoined code.
    
    Reviewers: Lucas Brutschy <lb...@confluent.io>
---
 .../internals/SlidingWindowStoreMaterializer.java  |  1 +
 .../internals/StreamJoinedStoreFactory.java        |  1 +
 .../kstream/internals/WindowStoreMaterializer.java |  3 ++-
 .../streams/state/BuiltInDslStoreSuppliers.java    | 18 ++++++++++----
 .../kafka/streams/state/DslWindowParams.java       | 10 +++++++-
 .../kstream/internals/KStreamKStreamJoinTest.java  | 28 ++++++++++++++++++++++
 6 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SlidingWindowStoreMaterializer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SlidingWindowStoreMaterializer.java
index 2e2e217f35d..10bae1ce606 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SlidingWindowStoreMaterializer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SlidingWindowStoreMaterializer.java
@@ -65,6 +65,7 @@ public class SlidingWindowStoreMaterializer<K, V> extends MaterializedStoreFacto
                         Duration.ofMillis(windows.timeDifferenceMs()),
                         false,
                         emitStrategy,
+                        true,
                         true
                 ))
                 : (WindowBytesStoreSupplier) materialized.storeSupplier();
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedStoreFactory.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedStoreFactory.java
index e8b7bc98e3d..a844621ba19 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedStoreFactory.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedStoreFactory.java
@@ -88,6 +88,7 @@ public class StreamJoinedStoreFactory<K, V1, V2> extends AbstractConfigurableSto
                         Duration.ofMillis(windows.size()),
                         true,
                         EmitStrategy.onWindowUpdate(),
+                        false,
                         false
                 ))
                 : storeSupplier;
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowStoreMaterializer.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowStoreMaterializer.java
index 25615875853..868a6b3aa3d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowStoreMaterializer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/WindowStoreMaterializer.java
@@ -63,7 +63,8 @@ public class WindowStoreMaterializer<K, V> extends MaterializedStoreFactory<K, V
                         Duration.ofMillis(windows.size()),
                         false,
                         emitStrategy,
-                        false
+                        false,
+                        true
                 ))
                 : (WindowBytesStoreSupplier) materialized.storeSupplier();
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/BuiltInDslStoreSuppliers.java b/streams/src/main/java/org/apache/kafka/streams/state/BuiltInDslStoreSuppliers.java
index 731b7ece7f8..71e7ac869fc 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/BuiltInDslStoreSuppliers.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/BuiltInDslStoreSuppliers.java
@@ -50,11 +50,19 @@ public class BuiltInDslStoreSuppliers {
                         params.isSlidingWindow());
             }
 
-            return Stores.persistentTimestampedWindowStore(
-                    params.name(),
-                    params.retentionPeriod(),
-                    params.windowSize(),
-                    params.retainDuplicates());
+            if (params.isTimestamped()) {
+                return Stores.persistentTimestampedWindowStore(
+                        params.name(),
+                        params.retentionPeriod(),
+                        params.windowSize(),
+                        params.retainDuplicates());
+            } else {
+                return Stores.persistentWindowStore(
+                        params.name(),
+                        params.retentionPeriod(),
+                        params.windowSize(),
+                        params.retainDuplicates());
+            }
         }
 
         @Override
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/DslWindowParams.java b/streams/src/main/java/org/apache/kafka/streams/state/DslWindowParams.java
index 88cc24b6a41..672afc66e0d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/DslWindowParams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/DslWindowParams.java
@@ -32,6 +32,7 @@ public class DslWindowParams {
     private final boolean retainDuplicates;
     private final EmitStrategy emitStrategy;
     private final boolean isSlidingWindow;
+    private final boolean isTimestamped;
 
     /**
      * @param name             name of the store (cannot be {@code null})
@@ -44,6 +45,7 @@ public class DslWindowParams {
      *                         caching and means that null values will be ignored.
      * @param emitStrategy     defines how to emit results
      * @param isSlidingWindow  whether the requested store is a sliding window
+     * @param isTimestamped    whether the requested store should be timestamped (see {@link TimestampedWindowStore}
      */
     public DslWindowParams(
             final String name,
@@ -51,8 +53,10 @@ public class DslWindowParams {
             final Duration windowSize,
             final boolean retainDuplicates,
             final EmitStrategy emitStrategy,
-            final boolean isSlidingWindow
+            final boolean isSlidingWindow,
+            final boolean isTimestamped
     ) {
+        this.isTimestamped = isTimestamped;
         Objects.requireNonNull(name);
         this.name = name;
         this.retentionPeriod = retentionPeriod;
@@ -86,6 +90,10 @@ public class DslWindowParams {
         return isSlidingWindow;
     }
 
+    public boolean isTimestamped() {
+        return isTimestamped;
+    }
+
     @Override
     public boolean equals(final Object o) {
         if (this == o) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
index 789cc54ace2..7fd62bb78bd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamKStreamJoinTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.streams.kstream.internals;
 
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.Serdes;
 import org.apache.kafka.common.serialization.StringSerializer;
@@ -53,6 +54,7 @@ import org.apache.kafka.streams.state.internals.WindowStoreBuilder;
 import org.apache.kafka.streams.state.internals.LeftOrRightValueSerde;
 import org.apache.kafka.streams.state.internals.LeftOrRightValue;
 import org.apache.kafka.streams.state.internals.InMemoryWindowBytesStoreSupplier;
+import org.apache.kafka.streams.state.internals.WrappedStateStore;
 import org.apache.kafka.test.MockApiProcessor;
 import org.apache.kafka.test.MockApiProcessorSupplier;
 import org.apache.kafka.test.MockValueJoiner;
@@ -422,6 +424,32 @@ public class KStreamKStreamJoinTest {
         assertThat(TrackingDslStoreSuppliers.NUM_CALLS.get(), is(2));
     }
 
+    public static class CapturingStoreSuppliers extends BuiltInDslStoreSuppliers.RocksDBDslStoreSuppliers {
+
+        final AtomicReference<WindowBytesStoreSupplier> capture = new AtomicReference<>();
+
+        @Override
+        public WindowBytesStoreSupplier windowStore(final DslWindowParams params) {
+            final WindowBytesStoreSupplier store = super.windowStore(params);
+            capture.set(store);
+            return store;
+        }
+    }
+
+    @Test
+    public void shouldJoinWithNonTimestampedStore() {
+        final JoinWindows joinWindows = JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100L));
+
+        final CapturingStoreSuppliers storeSuppliers = new CapturingStoreSuppliers();
+        final StreamJoined<String, Integer, Integer> streamJoined =
+                StreamJoined.with(Serdes.String(), Serdes.Integer(), Serdes.Integer())
+                        .withDslStoreSuppliers(storeSuppliers);
+
+        runJoin(streamJoined, joinWindows);
+        assertThat("Expected stream joined to supply builders that create non-timestamped stores",
+                !WrappedStateStore.isTimestamped(storeSuppliers.capture.get().get()));
+    }
+
     @Test
     public void shouldThrottleEmitNonJoinedOuterRecordsEvenWhenClockDrift() {
         /**