You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by gu...@apache.org on 2019/03/28 20:57:24 UTC

[kafka] branch trunk updated: KAFKA-7502: Cleanup KTable materialization logic in a single place (filter) (#6453)

This is an automated email from the ASF dual-hosted git repository.

guozhang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new d63d702  KAFKA-7502: Cleanup KTable materialization logic in a single place (filter) (#6453)
d63d702 is described below

commit d63d7022526565a96323a45fad4ead6c3db573ad
Author: Lee Dongjin <do...@apache.org>
AuthorDate: Fri Mar 29 05:57:04 2019 +0900

    KAFKA-7502: Cleanup KTable materialization logic in a single place (filter) (#6453)
    
    This PR is a follow-up of #6174, which handles doFilter / doMapValues / doTransformValues methods.
    
    Reviewers: Bill Bejeck <bi...@confluent.io>, Guozhang Wang <gu...@confluent.io>
---
 .../streams/kstream/internals/KTableFilter.java    |  3 +-
 .../streams/kstream/internals/KTableImpl.java      | 66 ++++++++++++++--------
 .../internals/graph/TableProcessorNode.java        | 42 +++++++-------
 3 files changed, 62 insertions(+), 49 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableFilter.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableFilter.java
index 1312e2f..2410074 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableFilter.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableFilter.java
@@ -71,7 +71,8 @@ class KTableFilter<K, V> implements KTableProcessorSupplier<K, V, V> {
             super.init(context);
             if (queryableName != null) {
                 store = (KeyValueStore<K, V>) context.getStateStore(queryableName);
-                tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context), sendOldValues);
+                tupleForwarder = new TupleForwarder<>(store, context,
+                    new ForwardingCacheFlushListener<>(context), sendOldValues);
             }
         }
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index d972136..67b3c31 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -114,17 +114,36 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
     private KTable<K, V> doFilter(final Predicate<? super K, ? super V> predicate,
                                   final MaterializedInternal<K, V, KeyValueStore<Bytes, byte[]>> materializedInternal,
                                   final boolean filterNot) {
-        // we actually do not need to generate store names at all since if it is not specified, we will not
-        // materialize the store; but we still need to burn one index BEFORE generating the processor to keep compatibility.
-        if (materializedInternal != null && materializedInternal.storeName() == null) {
-            builder.newStoreName(FILTER_NAME);
+        final Serde<K> keySerde;
+        final Serde<V> valueSerde;
+        final String queryableStoreName;
+        final StoreBuilder<KeyValueStore<K, V>> storeBuilder;
+
+        if (materializedInternal != null) {
+            // we actually do not need to generate store names at all since if it is not specified, we will not
+            // materialize the store; but we still need to burn one index BEFORE generating the processor to keep compatibility.
+            if (materializedInternal.storeName() == null) {
+                builder.newStoreName(FILTER_NAME);
+            }
+            // we can inherit parent key and value serde if user do not provide specific overrides, more specifically:
+            // we preserve the key following the order of 1) materialized, 2) parent
+            keySerde = materializedInternal.keySerde() != null ? materializedInternal.keySerde() : this.keySerde;
+            // we preserve the value following the order of 1) materialized, 2) parent
+            valueSerde = materializedInternal.valueSerde() != null ? materializedInternal.valueSerde() : this.valSerde;
+            queryableStoreName = materializedInternal.queryableStoreName();
+            // only materialize if materialized is specified and it has queryable name
+            storeBuilder = queryableStoreName != null ? (new KeyValueStoreMaterializer<>(materializedInternal)).materialize() : null;
+        } else {
+            keySerde = this.keySerde;
+            valueSerde = this.valSerde;
+            queryableStoreName = null;
+            storeBuilder = null;
         }
 
         final String name = builder.newProcessorName(FILTER_NAME);
 
-        // only materialize if the state store has queryable name
-        final String queryableName = materializedInternal != null ? materializedInternal.queryableStoreName() : null;
-        final KTableProcessorSupplier<K, V, V> processorSupplier = new KTableFilter<>(this, predicate, filterNot, queryableName);
+        final KTableProcessorSupplier<K, V, V> processorSupplier =
+            new KTableFilter<>(this, predicate, filterNot, queryableStoreName);
 
         final ProcessorParameters<K, V> processorParameters = unsafeCastProcessorParametersToCompletelyDifferentType(
             new ProcessorParameters<>(processorSupplier, name)
@@ -133,20 +152,16 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
         final StreamsGraphNode tableNode = new TableProcessorNode<>(
             name,
             processorParameters,
-            materializedInternal,
-            null
+            storeBuilder
         );
 
         builder.addGraphNode(this.streamsGraphNode, tableNode);
 
-        // we can inherit parent key and value serde if user do not provide specific overrides, more specifically:
-        // we preserve the key following the order of 1) materialized, 2) parent
-        // we preserve the value following the order of 1) materialized, 2) parent
         return new KTableImpl<>(name,
-                                materializedInternal != null && materializedInternal.keySerde() != null ? materializedInternal.keySerde() : keySerde,
-                                materializedInternal != null && materializedInternal.valueSerde() != null ? materializedInternal.valueSerde() : valSerde,
+                                keySerde,
+                                valueSerde,
                                 sourceNodes,
-                                queryableName,
+                                queryableStoreName,
                                 processorSupplier,
                                 tableNode,
                                 builder);
@@ -195,8 +210,9 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
         final String name = builder.newProcessorName(MAPVALUES_NAME);
 
         // only materialize if the state store has queryable name
-        final String queryableName = materializedInternal != null ? materializedInternal.queryableStoreName() : null;
-        final KTableProcessorSupplier<K, V, VR> processorSupplier = new KTableMapValues<>(this, mapper, queryableName);
+        final String queryableStoreName = materializedInternal != null ? materializedInternal.queryableStoreName() : null;
+        final StoreBuilder<KeyValueStore<K, VR>> storeBuilder = queryableStoreName != null ? (new KeyValueStoreMaterializer<>(materializedInternal)).materialize() : null;
+        final KTableProcessorSupplier<K, V, VR> processorSupplier = new KTableMapValues<>(this, mapper, queryableStoreName);
 
         // leaving in calls to ITB until building topology with graph
 
@@ -206,8 +222,7 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
         final StreamsGraphNode tableNode = new TableProcessorNode<>(
             name,
             processorParameters,
-            materializedInternal,
-            null
+            storeBuilder
         );
 
         builder.addGraphNode(this.streamsGraphNode, tableNode);
@@ -220,7 +235,7 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
             materializedInternal != null && materializedInternal.keySerde() != null ? materializedInternal.keySerde() : keySerde,
             materializedInternal != null ? materializedInternal.valueSerde() : null,
             sourceNodes,
-            queryableName,
+            queryableStoreName,
             processorSupplier,
             tableNode,
             builder
@@ -277,14 +292,15 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
     }
 
     private <VR> KTable<K, VR> doTransformValues(final ValueTransformerWithKeySupplier<? super K, ? super V, ? extends VR> transformerSupplier,
-                                                 final MaterializedInternal<K, VR, KeyValueStore<Bytes, byte[]>> materialized,
+                                                 final MaterializedInternal<K, VR, KeyValueStore<Bytes, byte[]>> materializedInternal,
                                                  final String... stateStoreNames) {
         Objects.requireNonNull(stateStoreNames, "stateStoreNames");
 
         final String name = builder.newProcessorName(TRANSFORMVALUES_NAME);
 
         // only materialize if users provide a specific queryable name
-        final String queryableStoreName = materialized != null ? materialized.queryableStoreName() : null;
+        final String queryableStoreName = materializedInternal != null ? materializedInternal.queryableStoreName() : null;
+        final StoreBuilder<KeyValueStore<K, VR>> storeBuilder = queryableStoreName != null ? (new KeyValueStoreMaterializer<>(materializedInternal)).materialize() : null;
 
         final KTableProcessorSupplier<K, V, VR> processorSupplier = new KTableTransformValues<>(
             this,
@@ -298,7 +314,7 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
         final StreamsGraphNode tableNode = new TableProcessorNode<>(
             name,
             processorParameters,
-            materialized,
+            storeBuilder,
             stateStoreNames
         );
 
@@ -309,8 +325,8 @@ public class KTableImpl<K, S, V> extends AbstractStream<K, V> implements KTable<
         // we preserve the value following the order of 1) materialized, 2) null
         return new KTableImpl<>(
             name,
-            materialized != null && materialized.keySerde() != null ? materialized.keySerde() : keySerde,
-            materialized != null ? materialized.valueSerde() : null,
+            materializedInternal != null && materializedInternal.keySerde() != null ? materializedInternal.keySerde() : keySerde,
+            materializedInternal != null ? materializedInternal.valueSerde() : null,
             sourceNodes,
             queryableStoreName,
             processorSupplier,
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java
index f335843..df009df 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/TableProcessorNode.java
@@ -17,39 +17,41 @@
 
 package org.apache.kafka.streams.kstream.internals.graph;
 
-import org.apache.kafka.common.utils.Bytes;
-import org.apache.kafka.streams.kstream.internals.KeyValueStoreMaterializer;
-import org.apache.kafka.streams.kstream.internals.MaterializedInternal;
-import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder;
 import org.apache.kafka.streams.state.KeyValueStore;
+import org.apache.kafka.streams.state.StoreBuilder;
 
 import java.util.Arrays;
 
-public class TableProcessorNode<K, V, S extends StateStore> extends StreamsGraphNode {
+public class TableProcessorNode<K, V> extends StreamsGraphNode {
 
-    private final MaterializedInternal<K, V, S> materializedInternal;
     private final ProcessorParameters<K, V> processorParameters;
+    private final StoreBuilder<KeyValueStore<K, V>> storeBuilder;
     private final String[] storeNames;
 
     public TableProcessorNode(final String nodeName,
                               final ProcessorParameters<K, V> processorParameters,
-                              final MaterializedInternal<K, V, S> materializedInternal,
-                              final String[] storeNames) {
+                              final StoreBuilder<KeyValueStore<K, V>> storeBuilder) {
+        this(nodeName, processorParameters, storeBuilder, null);
+    }
 
+    public TableProcessorNode(final String nodeName,
+                              final ProcessorParameters<K, V> processorParameters,
+                              final StoreBuilder<KeyValueStore<K, V>> storeBuilder,
+                              final String[] storeNames) {
         super(nodeName);
         this.processorParameters = processorParameters;
-        this.materializedInternal = materializedInternal;
-        this.storeNames = storeNames != null ? storeNames : new String[]{};
+        this.storeBuilder = storeBuilder;
+        this.storeNames = storeNames != null ? storeNames : new String[] {};
     }
 
     @Override
     public String toString() {
         return "TableProcessorNode{" +
-               "materializedInternal=" + materializedInternal +
-               ", processorParameters=" + processorParameters +
-               ", storeNames=" + Arrays.toString(storeNames) +
-               "} " + super.toString();
+            ", processorParameters=" + processorParameters +
+            ", storeBuilder=" + storeBuilder.name() +
+            ", storeNames=" + Arrays.toString(storeNames) +
+            "} " + super.toString();
     }
 
     @SuppressWarnings("unchecked")
@@ -62,15 +64,9 @@ public class TableProcessorNode<K, V, S extends StateStore> extends StreamsGraph
             topologyBuilder.connectProcessorAndStateStores(processorName, storeNames);
         }
 
-        // only materialize if materialized is specified and it is queryable
-        final boolean shouldMaterialize = materializedInternal != null && materializedInternal.queryableStoreName() != null;
-        if (shouldMaterialize) {
-            // TODO: we are enforcing this as a keyvalue store, but it should go beyond any type of stores
-            topologyBuilder.addStateStore(
-                new KeyValueStoreMaterializer<>(
-                    (MaterializedInternal<K, V, KeyValueStore<Bytes, byte[]>>) materializedInternal
-                ).materialize(),
-                processorName);
+        // TODO: we are enforcing this as a keyvalue store, but it should go beyond any type of stores
+        if (storeBuilder != null) {
+            topologyBuilder.addStateStore(storeBuilder, processorName);
         }
     }
 }