You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ab...@apache.org on 2021/07/21 20:32:17 UTC

[kafka] branch 3.0 updated: KAFKA-13096: Ensure queryable store providers is up to date after adding stream thread (#10921)

This is an automated email from the ASF dual-hosted git repository.

ableegoldman pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/3.0 by this push:
     new 949ff25  KAFKA-13096: Ensure queryable store providers is up to date after adding stream thread (#10921)
949ff25 is described below

commit 949ff251b4a022341a656f904a2a284cde988ede
Author: Phil Hardwick <ph...@gmail.com>
AuthorDate: Wed Jul 21 21:23:15 2021 +0100

    KAFKA-13096: Ensure queryable store providers is up to date after adding stream thread (#10921)
    
    When a new thread is added the queryable store providers continues to use the store providers it was given when KafkaStreams was instantiated. This means IQ will start performing lookups against an out-of-date list of threads, and may eventually become completely broken. We must make sure the QueryableStoreProvider is updated when threads are added and removed.
    
    Reviewers: Anna Sophie Blee-Goldman <ab...@apache.org>
---
 .../org/apache/kafka/streams/KafkaStreams.java     |  9 +--
 .../state/internals/QueryableStoreProvider.java    | 23 ++++--
 .../state/internals/WrappingStoreProvider.java     |  5 +-
 .../integration/StoreQueryIntegrationTest.java     | 86 ++++++++++++++++++++++
 .../internals/QueryableStoreProviderTest.java      |  3 +-
 5 files changed, 111 insertions(+), 15 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index 557a2b3..b508396 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -157,7 +157,6 @@ public class KafkaStreams implements AutoCloseable {
     private final StreamsMetadataState streamsMetadataState;
     private final ScheduledExecutorService stateDirCleaner;
     private final ScheduledExecutorService rocksDBMetricsRecordingService;
-    private final QueryableStoreProvider queryableStoreProvider;
     private final Admin adminClient;
     private final StreamsMetricsImpl streamsMetrics;
     private final ProcessorTopology taskTopology;
@@ -166,10 +165,10 @@ public class KafkaStreams implements AutoCloseable {
     private final StreamStateListener streamStateListener;
     private final StateRestoreListener delegatingStateRestoreListener;
     private final Map<Long, StreamThread.State> threadState;
-    private final ArrayList<StreamThreadStateStoreProvider> storeProviders;
     private final UUID processId;
     private final KafkaClientSupplier clientSupplier;
     private final InternalTopologyBuilder internalTopologyBuilder;
+    private final QueryableStoreProvider queryableStoreProvider;
 
     GlobalStreamThread globalStreamThread;
     private KafkaStreams.StateListener stateListener;
@@ -904,11 +903,10 @@ public class KafkaStreams implements AutoCloseable {
             globalStreamThread.setStateListener(streamStateListener);
         }
 
-        storeProviders = new ArrayList<>();
+        queryableStoreProvider = new QueryableStoreProvider(globalStateStoreProvider);
         for (int i = 1; i <= numStreamThreads; i++) {
             createAndAddStreamThread(cacheSizePerThread, i);
         }
-        queryableStoreProvider = new QueryableStoreProvider(storeProviders, globalStateStoreProvider);
 
         stateDirCleaner = setupStateDirCleaner();
         rocksDBMetricsRecordingService = maybeCreateRocksDBMetricsRecordingService(clientId, config);
@@ -935,7 +933,7 @@ public class KafkaStreams implements AutoCloseable {
         streamThread.setStateListener(streamStateListener);
         threads.add(streamThread);
         threadState.put(streamThread.getId(), streamThread.state());
-        storeProviders.add(new StreamThreadStateStoreProvider(streamThread));
+        queryableStoreProvider.addStoreProviderForThread(streamThread.getName(), new StreamThreadStateStoreProvider(streamThread));
         return streamThread;
     }
 
@@ -1081,6 +1079,7 @@ public class KafkaStreams implements AutoCloseable {
                             } else {
                                 log.info("Successfully removed {} in {}ms", streamThread.getName(), time.milliseconds() - startMs);
                                 threads.remove(streamThread);
+                                queryableStoreProvider.removeStoreProviderForThread(streamThread.getName());
                             }
                         } else {
                             log.info("{} is the last remaining thread and must remove itself, therefore we cannot wait "
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/QueryableStoreProvider.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/QueryableStoreProvider.java
index 8dd1f03..07cf0ee 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/QueryableStoreProvider.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/QueryableStoreProvider.java
@@ -20,20 +20,23 @@ import org.apache.kafka.streams.StoreQueryParameters;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.state.QueryableStoreType;
 
-import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * A wrapper over all of the {@link StateStoreProvider}s in a Topology
+ *
+ * The store providers field is a reference
  */
 public class QueryableStoreProvider {
 
-    private final List<StreamThreadStateStoreProvider> storeProviders;
+    // map of StreamThread.name to StreamThreadStateStoreProvider
+    private final Map<String, StreamThreadStateStoreProvider> storeProviders;
     private final GlobalStateStoreProvider globalStoreProvider;
 
-    public QueryableStoreProvider(final List<StreamThreadStateStoreProvider> storeProviders,
-                                  final GlobalStateStoreProvider globalStateStoreProvider) {
-        this.storeProviders = new ArrayList<>(storeProviders);
+    public QueryableStoreProvider(final GlobalStateStoreProvider globalStateStoreProvider) {
+        this.storeProviders = new HashMap<>();
         this.globalStoreProvider = globalStateStoreProvider;
     }
 
@@ -56,8 +59,16 @@ public class QueryableStoreProvider {
             return queryableStoreType.create(globalStoreProvider, storeName);
         }
         return queryableStoreType.create(
-            new WrappingStoreProvider(storeProviders, storeQueryParameters),
+            new WrappingStoreProvider(storeProviders.values(), storeQueryParameters),
             storeName
         );
     }
+
+    public void addStoreProviderForThread(final String threadName, final StreamThreadStateStoreProvider streamThreadStateStoreProvider) {
+        this.storeProviders.put(threadName, streamThreadStateStoreProvider);
+    }
+
+    public void removeStoreProviderForThread(final String threadName) {
+        this.storeProviders.remove(threadName);
+    }
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappingStoreProvider.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappingStoreProvider.java
index 03ac0ae..6b4ae92 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappingStoreProvider.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WrappingStoreProvider.java
@@ -22,6 +22,7 @@ import org.apache.kafka.streams.errors.InvalidStateStorePartitionException;
 import org.apache.kafka.streams.state.QueryableStoreType;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
 /**
@@ -29,10 +30,10 @@ import java.util.List;
  */
 public class WrappingStoreProvider implements StateStoreProvider {
 
-    private final List<StreamThreadStateStoreProvider> storeProviders;
+    private final Collection<StreamThreadStateStoreProvider> storeProviders;
     private StoreQueryParameters storeQueryParameters;
 
-    WrappingStoreProvider(final List<StreamThreadStateStoreProvider> storeProviders,
+    WrappingStoreProvider(final Collection<StreamThreadStateStoreProvider> storeProviders,
                           final StoreQueryParameters storeQueryParameters) {
         this.storeProviders = storeProviders;
         this.storeQueryParameters = storeQueryParameters;
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StoreQueryIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StoreQueryIntegrationTest.java
index d3e4f8c..d764ed7 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StoreQueryIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StoreQueryIntegrationTest.java
@@ -53,6 +53,7 @@ import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
@@ -67,6 +68,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertThrows;
@@ -412,6 +414,90 @@ public class StoreQueryIntegrationTest {
         assertThat(store4.get(key), is(nullValue()));
     }
 
+    @Test
+    public void shouldQueryStoresAfterAddingAndRemovingStreamThread() throws Exception {
+        final int batch1NumMessages = 100;
+        final int key = 1;
+        final int key2 = 2;
+        final int key3 = 3;
+        final Semaphore semaphore = new Semaphore(0);
+        final int numStreamThreads = 1;
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        builder.table(INPUT_TOPIC_NAME, Consumed.with(Serdes.Integer(), Serdes.Integer()),
+                      Materialized.<Integer, Integer, KeyValueStore<Bytes, byte[]>>as(TABLE_NAME)
+                          .withCachingDisabled())
+               .toStream()
+               .peek((k, v) -> {
+                   if (k.equals(key3)) {
+                       semaphore.release();
+                   }
+               });
+
+        final Properties streamsConfiguration1 = streamsConfiguration();
+        streamsConfiguration1.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, numStreamThreads);
+
+        final KafkaStreams kafkaStreams1 = createKafkaStreams(builder, streamsConfiguration1);
+        final List<KafkaStreams> kafkaStreamsList = Arrays.asList(kafkaStreams1);
+
+        startApplicationAndWaitUntilRunning(kafkaStreamsList, Duration.ofSeconds(60));
+        //Add thread
+        final Optional<String> streamThread = kafkaStreams1.addStreamThread();
+        assertThat(streamThread.isPresent(), is(true));
+
+        produceValueRange(key, 0, batch1NumMessages);
+        produceValueRange(key2, 0, batch1NumMessages);
+        produceValueRange(key3, 0, batch1NumMessages);
+
+        // Assert that all messages in the batches were processed in a timely manner
+        assertThat(semaphore.tryAcquire(batch1NumMessages, 60, TimeUnit.SECONDS), is(equalTo(true)));
+
+        until(() -> {
+            final QueryableStoreType<ReadOnlyKeyValueStore<Integer, Integer>> queryableStoreType = keyValueStore();
+            final ReadOnlyKeyValueStore<Integer, Integer> store1 = getStore(TABLE_NAME, kafkaStreams1, queryableStoreType);
+
+            try {
+                assertThat(store1.get(key), is(notNullValue()));
+                assertThat(store1.get(key2), is(notNullValue()));
+                assertThat(store1.get(key3), is(notNullValue()));
+                return true;
+            } catch (final InvalidStateStoreException exception) {
+                assertThat(
+                        exception.getMessage(),
+                        matchesRegex("Cannot get state store source-table because the stream thread is (PARTITIONS_ASSIGNED|STARTING|PARTITIONS_REVOKED), not RUNNING")
+                );
+                LOG.info("Streams wasn't running. Will try again.");
+                return false;
+            }
+        });
+
+        final Optional<String> removedThreadName = kafkaStreams1.removeStreamThread();
+        assertThat(removedThreadName.isPresent(), is(true));
+
+        until(() -> {
+            return kafkaStreams1.state().equals(KafkaStreams.State.RUNNING);
+        });
+
+        until(() -> {
+            final QueryableStoreType<ReadOnlyKeyValueStore<Integer, Integer>> queryableStoreType = keyValueStore();
+            final ReadOnlyKeyValueStore<Integer, Integer> store1 = getStore(TABLE_NAME, kafkaStreams1, queryableStoreType);
+
+            try {
+                assertThat(store1.get(key), is(notNullValue()));
+                assertThat(store1.get(key2), is(notNullValue()));
+                assertThat(store1.get(key3), is(notNullValue()));
+                return true;
+            } catch (final InvalidStateStoreException exception) {
+                assertThat(
+                        exception.getMessage(),
+                        matchesRegex("Cannot get state store source-table because the stream thread is (PARTITIONS_ASSIGNED|STARTING|PARTITIONS_REVOKED), not RUNNING")
+                );
+                LOG.info("Streams wasn't running. Will try again.");
+                return false;
+            }
+        });
+    }
+
     private static void until(final TestCondition condition) {
         boolean success = false;
         final long deadline = System.currentTimeMillis() + IntegrationTestUtils.DEFAULT_TIMEOUT;
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/QueryableStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/QueryableStoreProviderTest.java
index 6c9252e..79cfb65 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/QueryableStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/QueryableStoreProviderTest.java
@@ -27,7 +27,6 @@ import org.apache.kafka.test.StateStoreProviderStub;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.Collections;
 import java.util.HashMap;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -53,9 +52,9 @@ public class QueryableStoreProviderTest {
         globalStateStores = new HashMap<>();
         storeProvider =
             new QueryableStoreProvider(
-                Collections.singletonList(theStoreProvider),
                 new GlobalStateStoreProvider(globalStateStores)
             );
+        storeProvider.addStoreProviderForThread("thread1", theStoreProvider);
     }
 
     @Test