You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by gu...@apache.org on 2016/09/08 01:02:27 UTC
[1/2] kafka git commit: KAFKA-3595: window stores use compact,
delete config for changelogs
Repository: kafka
Updated Branches:
refs/heads/trunk eba0ede87 -> 69ebf6f7b
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
index b9a1cf6..2d9b9a5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java
@@ -35,10 +35,12 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KStreamBuilder;
+import org.apache.kafka.streams.kstream.TimeWindows;
import org.apache.kafka.streams.kstream.ValueMapper;
import org.apache.kafka.streams.processor.internals.ProcessorStateManager;
import org.apache.kafka.test.MockKeyValueMapper;
import org.apache.kafka.test.TestUtils;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -50,8 +52,10 @@ import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
+import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Tests related to internal topics in streams
@@ -65,6 +69,8 @@ public class InternalTopicIntegrationTest {
private static final String DEFAULT_OUTPUT_TOPIC = "outputTopic";
private static final int DEFAULT_ZK_SESSION_TIMEOUT_MS = 10 * 1000;
private static final int DEFAULT_ZK_CONNECTION_TIMEOUT_MS = 8 * 1000;
+ private Properties streamsConfiguration;
+ private String applicationId = "compact-topics-integration-test";
@BeforeClass
public static void startKafkaCluster() throws Exception {
@@ -72,14 +78,20 @@ public class InternalTopicIntegrationTest {
CLUSTER.createTopic(DEFAULT_OUTPUT_TOPIC);
}
- /**
- * Validates that any state changelog topics are compacted
- *
- * @return true if topics have a valid config, false otherwise
- */
- private boolean isUsingCompactionForStateChangelogTopics() {
- boolean valid = true;
+ @Before
+ public void before() {
+ streamsConfiguration = new Properties();
+ streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
+ streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+ streamsConfiguration.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, CLUSTER.zKConnectString());
+ streamsConfiguration.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+ streamsConfiguration.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
+ streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
+ streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ }
+
+ private Properties getTopicConfigProperties(final String changelog) {
// Note: You must initialize the ZkClient with ZKStringSerializer. If you don't, then
// createTopic() will only seem to work (it will return without error). The topic will exist in
// only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
@@ -89,33 +101,28 @@ public class InternalTopicIntegrationTest {
DEFAULT_ZK_SESSION_TIMEOUT_MS,
DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
ZKStringSerializer$.MODULE$);
- final boolean isSecure = false;
- final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(CLUSTER.zKConnectString()), isSecure);
-
- final Map<String, Properties> topicConfigs = AdminUtils.fetchAllTopicConfigs(zkUtils);
- final Iterator it = topicConfigs.iterator();
- while (it.hasNext()) {
- final Tuple2<String, Properties> topicConfig = (Tuple2<String, Properties>) it.next();
- final String topic = topicConfig._1;
- final Properties prop = topicConfig._2;
-
- // state changelogs should be compacted
- if (topic.endsWith(ProcessorStateManager.STATE_CHANGELOG_TOPIC_SUFFIX)) {
- if (!prop.containsKey(LogConfig.CleanupPolicyProp()) ||
- !prop.getProperty(LogConfig.CleanupPolicyProp()).equals(LogConfig.Compact())) {
- valid = false;
- break;
+ try {
+ final boolean isSecure = false;
+ final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(CLUSTER.zKConnectString()), isSecure);
+
+ final Map<String, Properties> topicConfigs = AdminUtils.fetchAllTopicConfigs(zkUtils);
+ final Iterator it = topicConfigs.iterator();
+ while (it.hasNext()) {
+ final Tuple2<String, Properties> topicConfig = (Tuple2<String, Properties>) it.next();
+ final String topic = topicConfig._1;
+ final Properties prop = topicConfig._2;
+ if (topic.equals(changelog)) {
+ return prop;
}
}
+ return new Properties();
+ } finally {
+ zkClient.close();
}
- zkClient.close();
- return valid;
}
@Test
public void shouldCompactTopicsForStateChangelogs() throws Exception {
- final List<String> inputValues = Arrays.asList("hello", "world", "world", "hello world");
-
//
// Step 1: Configure and start a simple word count topology
//
@@ -154,6 +161,17 @@ public class InternalTopicIntegrationTest {
//
// Step 2: Produce some input data to the input topic.
//
+ produceData(Arrays.asList("hello", "world", "world", "hello world"));
+
+ //
+ // Step 3: Verify the state changelog topics are compact
+ //
+ streams.close();
+ final Properties properties = getTopicConfigProperties(ProcessorStateManager.storeChangelogTopic(applicationId, "Counts"));
+ assertEquals(LogConfig.Compact(), properties.getProperty(LogConfig.CleanupPolicyProp()));
+ }
+
+ private void produceData(final List<String> inputValues) throws java.util.concurrent.ExecutionException, InterruptedException {
final Properties producerConfig = new Properties();
producerConfig.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
producerConfig.put(ProducerConfig.ACKS_CONFIG, "all");
@@ -161,11 +179,47 @@ public class InternalTopicIntegrationTest {
producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
IntegrationTestUtils.produceValuesSynchronously(DEFAULT_INPUT_TOPIC, inputValues, producerConfig, mockTime);
+ }
+
+ @Test
+ public void shouldUseCompactAndDeleteForWindowStoreChangelogs() throws Exception {
+ KStreamBuilder builder = new KStreamBuilder();
+
+ KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
+
+ final int durationMs = 2000;
+ textLines
+ .flatMapValues(new ValueMapper<String, Iterable<String>>() {
+ @Override
+ public Iterable<String> apply(String value) {
+ return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
+ }
+ }).groupBy(MockKeyValueMapper.<String, String>SelectValueMapper())
+ .count(TimeWindows.of(1000).until(durationMs), "CountWindows").toStream();
+
+
+ // Remove any state from previous test runs
+ IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+
+ KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
+ streams.start();
+
+ //
+ // Step 2: Produce some input data to the input topic.
+ //
+ produceData(Arrays.asList("hello", "world", "world", "hello world"));
//
// Step 3: Verify the state changelog topics are compact
//
streams.close();
- assertEquals(isUsingCompactionForStateChangelogTopics(), true);
+ final Properties properties = getTopicConfigProperties(ProcessorStateManager.storeChangelogTopic(applicationId, "CountWindows"));
+ final List<String> policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(","));
+ assertEquals(2, policies.size());
+ assertTrue(policies.contains(LogConfig.Compact()));
+ assertTrue(policies.contains(LogConfig.Delete()));
+ // retention should be 1 day + the window duration
+ final Long retention = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + durationMs;
+ assertEquals(retention, Long.valueOf(properties.getProperty(LogConfig.RetentionMsProp())));
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
index fe66acb..a4c008a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/TopologyBuilderTest.java
@@ -19,19 +19,24 @@ package org.apache.kafka.streams.processor;
import org.apache.kafka.streams.errors.TopologyBuilderException;
import org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo;
+import org.apache.kafka.streams.processor.internals.InternalTopicConfig;
+import org.apache.kafka.streams.processor.internals.InternalTopicManager;
import org.apache.kafka.streams.processor.internals.ProcessorNode;
import org.apache.kafka.streams.processor.internals.ProcessorStateManager;
import org.apache.kafka.streams.processor.internals.ProcessorTopology;
+import org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier;
import org.apache.kafka.test.MockProcessorSupplier;
import org.apache.kafka.test.MockStateStoreSupplier;
import org.junit.Test;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
@@ -264,9 +269,9 @@ public class TopologyBuilderTest {
Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
- expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "X-topic-1x", "topic-2"), Collections.<String>emptySet(), Collections.<String>emptySet()));
- expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String>emptySet(), Collections.<String>emptySet()));
- expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String>emptySet(), Collections.<String>emptySet()));
+ expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "X-topic-1x", "topic-2"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
+ expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
+ expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
assertEquals(3, topicGroups.size());
assertEquals(expectedTopicGroups, topicGroups);
@@ -302,9 +307,32 @@ public class TopologyBuilderTest {
Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
- expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "topic-1x", "topic-2"), Collections.<String>emptySet(), mkSet(ProcessorStateManager.storeChangelogTopic("X", "store-1"))));
- expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String>emptySet(), mkSet(ProcessorStateManager.storeChangelogTopic("X", "store-2"))));
- expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String>emptySet(), mkSet(ProcessorStateManager.storeChangelogTopic("X", "store-3"))));
+ final String store1 = ProcessorStateManager.storeChangelogTopic("X", "store-1");
+ final String store2 = ProcessorStateManager.storeChangelogTopic("X", "store-2");
+ final String store3 = ProcessorStateManager.storeChangelogTopic("X", "store-3");
+ expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "topic-1x", "topic-2"),
+ Collections.<String, InternalTopicConfig>emptyMap(),
+ Collections.singletonMap(store1,
+ new InternalTopicConfig(
+ store1,
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.compact),
+ Collections.<String, String>emptyMap()))));
+ expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"),
+ Collections.<String, InternalTopicConfig>emptyMap(),
+ Collections.singletonMap(store2,
+ new InternalTopicConfig(
+ store2,
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.compact),
+ Collections.<String, String>emptyMap()))));
+ expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"),
+ Collections.<String, InternalTopicConfig>emptyMap(),
+ Collections.singletonMap(store3,
+ new InternalTopicConfig(
+ store3,
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.compact),
+ Collections.<String, String>emptyMap()))));
+
+
assertEquals(3, topicGroups.size());
assertEquals(expectedTopicGroups, topicGroups);
@@ -390,7 +418,7 @@ public class TopologyBuilderTest {
@Test(expected = NullPointerException.class)
public void shouldNotAddNullStateStoreSupplier() throws Exception {
final TopologyBuilder builder = new TopologyBuilder();
- builder.addStateStore(null, true);
+ builder.addStateStore(null);
}
private Set<String> nodeNames(Collection<ProcessorNode> nodes) {
@@ -406,7 +434,7 @@ public class TopologyBuilderTest {
final TopologyBuilder builder = new TopologyBuilder();
builder.addSource("source", "topic");
builder.addProcessor("processor", new MockProcessorSupplier(), "source");
- builder.addStateStore(new MockStateStoreSupplier("store", false), true, "processor");
+ builder.addStateStore(new MockStateStoreSupplier("store", false), "processor");
final Map<String, Set<String>> stateStoreNameToSourceTopic = builder.stateStoreNameToSourceTopics();
assertEquals(1, stateStoreNameToSourceTopic.size());
assertEquals(Collections.singleton("topic"), stateStoreNameToSourceTopic.get("store"));
@@ -417,7 +445,7 @@ public class TopologyBuilderTest {
final TopologyBuilder builder = new TopologyBuilder();
builder.addSource("source", "topic");
builder.addProcessor("processor", new MockProcessorSupplier(), "source");
- builder.addStateStore(new MockStateStoreSupplier("store", false), false, "processor");
+ builder.addStateStore(new MockStateStoreSupplier("store", false), "processor");
final Map<String, Set<String>> stateStoreNameToSourceTopic = builder.stateStoreNameToSourceTopics();
assertEquals(1, stateStoreNameToSourceTopic.size());
assertEquals(Collections.singleton("topic"), stateStoreNameToSourceTopic.get("store"));
@@ -430,10 +458,63 @@ public class TopologyBuilderTest {
builder.addInternalTopic("internal-topic");
builder.addSource("source", "internal-topic");
builder.addProcessor("processor", new MockProcessorSupplier(), "source");
- builder.addStateStore(new MockStateStoreSupplier("store", false), true, "processor");
+ builder.addStateStore(new MockStateStoreSupplier("store", false), "processor");
final Map<String, Set<String>> stateStoreNameToSourceTopic = builder.stateStoreNameToSourceTopics();
assertEquals(1, stateStoreNameToSourceTopic.size());
assertEquals(Collections.singleton("appId-internal-topic"), stateStoreNameToSourceTopic.get("store"));
}
+ @SuppressWarnings("unchecked")
+ @Test
+ public void shouldAddInternalTopicConfigWithCompactAndDeleteSetForWindowStores() throws Exception {
+ final TopologyBuilder builder = new TopologyBuilder();
+ builder.setApplicationId("appId");
+ builder.addSource("source", "topic");
+ builder.addProcessor("processor", new MockProcessorSupplier(), "source");
+ builder.addStateStore(new RocksDBWindowStoreSupplier("store", 30000, 3, false, null, null, true, Collections.<String, String>emptyMap()), "processor");
+ final Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
+ final TopicsInfo topicsInfo = topicGroups.values().iterator().next();
+ final InternalTopicConfig topicConfig = topicsInfo.stateChangelogTopics.get("appId-store-changelog");
+ final Properties properties = topicConfig.toProperties(0);
+ final List<String> policies = Arrays.asList(properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP).split(","));
+ assertEquals("appId-store-changelog", topicConfig.name());
+ assertTrue(policies.contains("compact"));
+ assertTrue(policies.contains("delete"));
+ assertEquals(2, policies.size());
+ assertEquals("30000", properties.getProperty(InternalTopicManager.RETENTION_MS));
+ assertEquals(2, properties.size());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void shouldAddInternalTopicConfigWithCompactForNonWindowStores() throws Exception {
+ final TopologyBuilder builder = new TopologyBuilder();
+ builder.setApplicationId("appId");
+ builder.addSource("source", "topic");
+ builder.addProcessor("processor", new MockProcessorSupplier(), "source");
+ builder.addStateStore(new MockStateStoreSupplier("name", true), "processor");
+ final Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
+ final TopicsInfo topicsInfo = topicGroups.values().iterator().next();
+ final InternalTopicConfig topicConfig = topicsInfo.stateChangelogTopics.get("appId-name-changelog");
+ final Properties properties = topicConfig.toProperties(0);
+ assertEquals("appId-name-changelog", topicConfig.name());
+ assertEquals("compact", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
+ assertEquals(1, properties.size());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics() throws Exception {
+ final TopologyBuilder builder = new TopologyBuilder();
+ builder.setApplicationId("appId");
+ builder.addInternalTopic("foo");
+ builder.addSource("source", "foo");
+ final TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next();
+ final InternalTopicConfig topicConfig = topicsInfo.interSourceTopics.get("appId-foo");
+ final Properties properties = topicConfig.toProperties(0);
+ assertEquals("appId-foo", topicConfig.name());
+ assertEquals("delete", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
+ assertEquals(1, properties.size());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java
new file mode 100644
index 0000000..b0a198b
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.processor.internals;
+
+import org.apache.kafka.common.utils.Utils;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class InternalTopicConfigTest {
+
+ @Test
+ public void shouldHaveCompactionPropSetIfSupplied() throws Exception {
+ final Properties properties = new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.compact),
+ Collections.<String, String>emptyMap()).toProperties(0);
+ assertEquals("compact", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
+ }
+
+
+ @Test(expected = NullPointerException.class)
+ public void shouldThrowIfNameIsNull() throws Exception {
+ new InternalTopicConfig(null, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap());
+ }
+
+ @Test
+ public void shouldConfigureRetentionMsWithAdditionalRetentionWhenCompactAndDelete() throws Exception {
+ final InternalTopicConfig topicConfig = new InternalTopicConfig("name",
+ Utils.mkSet(InternalTopicConfig.CleanupPolicy.compact, InternalTopicConfig.CleanupPolicy.delete),
+ Collections.<String, String>emptyMap());
+ final int additionalRetentionMs = 20;
+ topicConfig.setRetentionMs(10);
+ final Properties properties = topicConfig.toProperties(additionalRetentionMs);
+ assertEquals("30", properties.getProperty(InternalTopicManager.RETENTION_MS));
+ }
+
+ @Test
+ public void shouldNotConfigureRetentionMsWhenCompact() throws Exception {
+ final InternalTopicConfig topicConfig = new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.compact),
+ Collections.<String, String>emptyMap());
+ topicConfig.setRetentionMs(10);
+ final Properties properties = topicConfig.toProperties(0);
+ assertNull(null, properties.getProperty(InternalTopicManager.RETENTION_MS));
+ }
+
+ @Test
+ public void shouldNotConfigureRetentionMsWhenDelete() throws Exception {
+ final InternalTopicConfig topicConfig = new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.delete),
+ Collections.<String, String>emptyMap());
+ topicConfig.setRetentionMs(10);
+ final Properties properties = topicConfig.toProperties(0);
+ assertNull(null, properties.getProperty(InternalTopicManager.RETENTION_MS));
+ }
+
+
+ @Test
+ public void shouldBeCompactedIfCleanupPolicyCompactOrCompactAndDelete() throws Exception {
+ assertTrue(new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.compact),
+ Collections.<String, String>emptyMap()).isCompacted());
+ assertTrue(new InternalTopicConfig("name", Utils.mkSet(InternalTopicConfig.CleanupPolicy.compact,
+ InternalTopicConfig.CleanupPolicy.delete),
+ Collections.<String, String>emptyMap()).isCompacted());
+ }
+
+ @Test
+ public void shouldNotBeCompactedWhenCleanupPolicyIsDelete() throws Exception {
+ assertFalse(new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.delete),
+ Collections.<String, String>emptyMap()).isCompacted());
+ }
+
+ @Test
+ public void shouldUseCleanupPolicyFromConfigIfSupplied() throws Exception {
+ final InternalTopicConfig config = new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.delete),
+ Collections.singletonMap("cleanup.policy", "compact"));
+
+ final Properties properties = config.toProperties(0);
+ assertEquals("compact", properties.getProperty("cleanup.policy"));
+ }
+
+ @Test
+ public void shouldHavePropertiesSuppliedByUser() throws Exception {
+ final Map<String, String> configs = new HashMap<>();
+ configs.put("retention.ms", "1000");
+ configs.put("retention.bytes", "10000");
+
+ final InternalTopicConfig topicConfig = new InternalTopicConfig("name",
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.delete),
+ configs);
+
+ final Properties properties = topicConfig.toProperties(0);
+ assertEquals("1000", properties.getProperty("retention.ms"));
+ assertEquals("10000", properties.getProperty("retention.bytes"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java
index e300966..e5ae7d8 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignorTest.java
@@ -735,15 +735,15 @@ public class StreamPartitionAssignorTest {
}
@Override
- public void makeReady(String topic, int numPartitions, boolean compactTopic) {
- readyTopics.put(topic, numPartitions);
+ public void makeReady(InternalTopicConfig topic, int numPartitions) {
+ readyTopics.put(topic.name(), numPartitions);
List<PartitionInfo> partitions = new ArrayList<>();
for (int i = 0; i < numPartitions; i++) {
- partitions.add(new PartitionInfo(topic, i, null, null, null));
+ partitions.add(new PartitionInfo(topic.name(), i, null, null, null));
}
- restoreConsumer.updatePartitions(topic, partitions);
+ restoreConsumer.updatePartitions(topic.name(), partitions);
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/state/StoresTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/StoresTest.java b/streams/src/test/java/org/apache/kafka/streams/state/StoresTest.java
new file mode 100644
index 0000000..18d158d
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/StoresTest.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.processor.StateStoreSupplier;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class StoresTest {
+
+ @Test
+ public void shouldCreateInMemoryStoreSupplierWithLoggedConfig() throws Exception {
+ final StateStoreSupplier supplier = Stores.create("store")
+ .withKeys(Serdes.String())
+ .withValues(Serdes.String())
+ .inMemory()
+ .enableLogging(Collections.singletonMap("retention.ms", "1000"))
+ .build();
+
+ final Map<String, String> config = supplier.logConfig();
+ assertTrue(supplier.loggingEnabled());
+ assertEquals("1000", config.get("retention.ms"));
+ }
+
+ @Test
+ public void shouldCreateInMemoryStoreSupplierNotLogged() throws Exception {
+ final StateStoreSupplier supplier = Stores.create("store")
+ .withKeys(Serdes.String())
+ .withValues(Serdes.String())
+ .inMemory()
+ .disableLogging()
+ .build();
+
+ assertFalse(supplier.loggingEnabled());
+ }
+
+ @Test
+ public void shouldCreatePersistenStoreSupplierWithLoggedConfig() throws Exception {
+ final StateStoreSupplier supplier = Stores.create("store")
+ .withKeys(Serdes.String())
+ .withValues(Serdes.String())
+ .persistent()
+ .enableLogging(Collections.singletonMap("retention.ms", "1000"))
+ .build();
+
+ final Map<String, String> config = supplier.logConfig();
+ assertTrue(supplier.loggingEnabled());
+ assertEquals("1000", config.get("retention.ms"));
+ }
+
+ @Test
+ public void shouldCreatePersistenStoreSupplierNotLogged() throws Exception {
+ final StateStoreSupplier supplier = Stores.create("store")
+ .withKeys(Serdes.String())
+ .withValues(Serdes.String())
+ .persistent()
+ .disableLogging()
+ .build();
+
+ assertFalse(supplier.loggingEnabled());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
index 8a22d37..700655e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/AbstractKeyValueStoreTest.java
@@ -26,125 +26,128 @@ import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.KeyValueStoreTestDriver;
+import org.apache.kafka.test.MockProcessorContext;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public abstract class AbstractKeyValueStoreTest {
+
+
protected abstract <K, V> KeyValueStore<K, V> createKeyValueStore(ProcessorContext context,
Class<K> keyClass, Class<V> valueClass,
boolean useContextSerdes);
+ protected KeyValueStore<Integer, String> store;
+ protected KeyValueStoreTestDriver<Integer, String> driver;
+
+ @Before
+ public void before() {
+ driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
+ final MockProcessorContext context = (MockProcessorContext) driver.context();
+ context.setTime(10);
+ store = createKeyValueStore(context, Integer.class, String.class, false);
+ }
+
+ @After
+ public void after() {
+ store.close();
+ }
+
@Test
public void testPutGetRange() {
- // Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, false);
- try {
-
- // Verify that the store reads and writes correctly ...
- store.put(0, "zero");
- store.put(1, "one");
- store.put(2, "two");
- store.put(4, "four");
- store.put(5, "five");
- assertEquals(5, driver.sizeOf(store));
- assertEquals("zero", store.get(0));
- assertEquals("one", store.get(1));
- assertEquals("two", store.get(2));
- assertNull(store.get(3));
- assertEquals("four", store.get(4));
- assertEquals("five", store.get(5));
- store.delete(5);
-
- // Flush the store and verify all current entries were properly flushed ...
- store.flush();
- assertEquals("zero", driver.flushedEntryStored(0));
- assertEquals("one", driver.flushedEntryStored(1));
- assertEquals("two", driver.flushedEntryStored(2));
- assertEquals("four", driver.flushedEntryStored(4));
- assertEquals(null, driver.flushedEntryStored(5));
-
- assertEquals(false, driver.flushedEntryRemoved(0));
- assertEquals(false, driver.flushedEntryRemoved(1));
- assertEquals(false, driver.flushedEntryRemoved(2));
- assertEquals(false, driver.flushedEntryRemoved(4));
- assertEquals(true, driver.flushedEntryRemoved(5));
-
- // Check range iteration ...
- try (KeyValueIterator<Integer, String> iter = store.range(2, 4)) {
- while (iter.hasNext()) {
- KeyValue<Integer, String> entry = iter.next();
- if (entry.key.equals(2))
- assertEquals("two", entry.value);
- else if (entry.key.equals(4))
- assertEquals("four", entry.value);
- else
- fail("Unexpected entry: " + entry);
- }
+ // Verify that the store reads and writes correctly ...
+ store.put(0, "zero");
+ store.put(1, "one");
+ store.put(2, "two");
+ store.put(4, "four");
+ store.put(5, "five");
+ assertEquals(5, driver.sizeOf(store));
+ assertEquals("zero", store.get(0));
+ assertEquals("one", store.get(1));
+ assertEquals("two", store.get(2));
+ assertNull(store.get(3));
+ assertEquals("four", store.get(4));
+ assertEquals("five", store.get(5));
+ store.delete(5);
+
+ // Flush the store and verify all current entries were properly flushed ...
+ store.flush();
+ assertEquals("zero", driver.flushedEntryStored(0));
+ assertEquals("one", driver.flushedEntryStored(1));
+ assertEquals("two", driver.flushedEntryStored(2));
+ assertEquals("four", driver.flushedEntryStored(4));
+ assertEquals(null, driver.flushedEntryStored(5));
+
+ assertEquals(false, driver.flushedEntryRemoved(0));
+ assertEquals(false, driver.flushedEntryRemoved(1));
+ assertEquals(false, driver.flushedEntryRemoved(2));
+ assertEquals(false, driver.flushedEntryRemoved(4));
+ assertEquals(true, driver.flushedEntryRemoved(5));
+
+ // Check range iteration ...
+ try (KeyValueIterator<Integer, String> iter = store.range(2, 4)) {
+ while (iter.hasNext()) {
+ KeyValue<Integer, String> entry = iter.next();
+ if (entry.key.equals(2))
+ assertEquals("two", entry.value);
+ else if (entry.key.equals(4))
+ assertEquals("four", entry.value);
+ else
+ fail("Unexpected entry: " + entry);
}
+ }
- // Check range iteration ...
- try (KeyValueIterator<Integer, String> iter = store.range(2, 6)) {
- while (iter.hasNext()) {
- KeyValue<Integer, String> entry = iter.next();
- if (entry.key.equals(2))
- assertEquals("two", entry.value);
- else if (entry.key.equals(4))
- assertEquals("four", entry.value);
- else
- fail("Unexpected entry: " + entry);
- }
+ // Check range iteration ...
+ try (KeyValueIterator<Integer, String> iter = store.range(2, 6)) {
+ while (iter.hasNext()) {
+ KeyValue<Integer, String> entry = iter.next();
+ if (entry.key.equals(2))
+ assertEquals("two", entry.value);
+ else if (entry.key.equals(4))
+ assertEquals("four", entry.value);
+ else
+ fail("Unexpected entry: " + entry);
}
- } finally {
- store.close();
}
}
@Test
public void testPutGetRangeWithDefaultSerdes() {
- // Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, true);
- try {
-
- // Verify that the store reads and writes correctly ...
- store.put(0, "zero");
- store.put(1, "one");
- store.put(2, "two");
- store.put(4, "four");
- store.put(5, "five");
- assertEquals(5, driver.sizeOf(store));
- assertEquals("zero", store.get(0));
- assertEquals("one", store.get(1));
- assertEquals("two", store.get(2));
- assertNull(store.get(3));
- assertEquals("four", store.get(4));
- assertEquals("five", store.get(5));
- store.delete(5);
-
- // Flush the store and verify all current entries were properly flushed ...
- store.flush();
- assertEquals("zero", driver.flushedEntryStored(0));
- assertEquals("one", driver.flushedEntryStored(1));
- assertEquals("two", driver.flushedEntryStored(2));
- assertEquals("four", driver.flushedEntryStored(4));
- assertEquals(null, driver.flushedEntryStored(5));
-
- assertEquals(false, driver.flushedEntryRemoved(0));
- assertEquals(false, driver.flushedEntryRemoved(1));
- assertEquals(false, driver.flushedEntryRemoved(2));
- assertEquals(false, driver.flushedEntryRemoved(4));
- assertEquals(true, driver.flushedEntryRemoved(5));
- } finally {
- store.close();
- }
+ // Verify that the store reads and writes correctly ...
+ store.put(0, "zero");
+ store.put(1, "one");
+ store.put(2, "two");
+ store.put(4, "four");
+ store.put(5, "five");
+ assertEquals(5, driver.sizeOf(store));
+ assertEquals("zero", store.get(0));
+ assertEquals("one", store.get(1));
+ assertEquals("two", store.get(2));
+ assertNull(store.get(3));
+ assertEquals("four", store.get(4));
+ assertEquals("five", store.get(5));
+ store.delete(5);
+
+ // Flush the store and verify all current entries were properly flushed ...
+ store.flush();
+ assertEquals("zero", driver.flushedEntryStored(0));
+ assertEquals("one", driver.flushedEntryStored(1));
+ assertEquals("two", driver.flushedEntryStored(2));
+ assertEquals("four", driver.flushedEntryStored(4));
+ assertEquals(null, driver.flushedEntryStored(5));
+
+ assertEquals(false, driver.flushedEntryRemoved(0));
+ assertEquals(false, driver.flushedEntryRemoved(1));
+ assertEquals(false, driver.flushedEntryRemoved(2));
+ assertEquals(false, driver.flushedEntryRemoved(4));
+ assertEquals(true, driver.flushedEntryRemoved(5));
}
@Test
public void testRestore() {
- // Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
-
+ store.close();
// Add any entries that will be restored to any store
// that uses the driver's context ...
driver.addEntryToRestoreLog(0, "zero");
@@ -154,23 +157,17 @@ public abstract class AbstractKeyValueStoreTest {
// Create the store, which should register with the context and automatically
// receive the restore entries ...
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, false);
- try {
- // Verify that the store's contents were properly restored ...
- assertEquals(0, driver.checkForRestoredEntries(store));
-
- // and there are no other entries ...
- assertEquals(4, driver.sizeOf(store));
- } finally {
- store.close();
- }
+ store = createKeyValueStore(driver.context(), Integer.class, String.class, false);
+ // Verify that the store's contents were properly restored ...
+ assertEquals(0, driver.checkForRestoredEntries(store));
+
+ // and there are no other entries ...
+ assertEquals(4, driver.sizeOf(store));
}
@Test
public void testRestoreWithDefaultSerdes() {
- // Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
-
+ store.close();
// Add any entries that will be restored to any store
// that uses the driver's context ...
driver.addEntryToRestoreLog(0, "zero");
@@ -180,70 +177,51 @@ public abstract class AbstractKeyValueStoreTest {
// Create the store, which should register with the context and automatically
// receive the restore entries ...
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, true);
- try {
- // Verify that the store's contents were properly restored ...
- assertEquals(0, driver.checkForRestoredEntries(store));
-
- // and there are no other entries ...
- assertEquals(4, driver.sizeOf(store));
- } finally {
- store.close();
- }
+ store = createKeyValueStore(driver.context(), Integer.class, String.class, true);
+ // Verify that the store's contents were properly restored ...
+ assertEquals(0, driver.checkForRestoredEntries(store));
+
+ // and there are no other entries ...
+ assertEquals(4, driver.sizeOf(store));
}
@Test
public void testPutIfAbsent() {
- // Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, true);
- try {
-
- // Verify that the store reads and writes correctly ...
- assertNull(store.putIfAbsent(0, "zero"));
- assertNull(store.putIfAbsent(1, "one"));
- assertNull(store.putIfAbsent(2, "two"));
- assertNull(store.putIfAbsent(4, "four"));
- assertEquals("four", store.putIfAbsent(4, "unexpected value"));
- assertEquals(4, driver.sizeOf(store));
- assertEquals("zero", store.get(0));
- assertEquals("one", store.get(1));
- assertEquals("two", store.get(2));
- assertNull(store.get(3));
- assertEquals("four", store.get(4));
-
- // Flush the store and verify all current entries were properly flushed ...
- store.flush();
- assertEquals("zero", driver.flushedEntryStored(0));
- assertEquals("one", driver.flushedEntryStored(1));
- assertEquals("two", driver.flushedEntryStored(2));
- assertEquals("four", driver.flushedEntryStored(4));
-
- assertEquals(false, driver.flushedEntryRemoved(0));
- assertEquals(false, driver.flushedEntryRemoved(1));
- assertEquals(false, driver.flushedEntryRemoved(2));
- assertEquals(false, driver.flushedEntryRemoved(4));
- } finally {
- store.close();
- }
+ // Verify that the store reads and writes correctly ...
+ assertNull(store.putIfAbsent(0, "zero"));
+ assertNull(store.putIfAbsent(1, "one"));
+ assertNull(store.putIfAbsent(2, "two"));
+ assertNull(store.putIfAbsent(4, "four"));
+ assertEquals("four", store.putIfAbsent(4, "unexpected value"));
+ assertEquals(4, driver.sizeOf(store));
+ assertEquals("zero", store.get(0));
+ assertEquals("one", store.get(1));
+ assertEquals("two", store.get(2));
+ assertNull(store.get(3));
+ assertEquals("four", store.get(4));
+
+ // Flush the store and verify all current entries were properly flushed ...
+ store.flush();
+ assertEquals("zero", driver.flushedEntryStored(0));
+ assertEquals("one", driver.flushedEntryStored(1));
+ assertEquals("two", driver.flushedEntryStored(2));
+ assertEquals("four", driver.flushedEntryStored(4));
+
+ assertEquals(false, driver.flushedEntryRemoved(0));
+ assertEquals(false, driver.flushedEntryRemoved(1));
+ assertEquals(false, driver.flushedEntryRemoved(2));
+ assertEquals(false, driver.flushedEntryRemoved(4));
}
@Test
public void testSize() {
- // Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, true);
- try {
- assertEquals("A newly created store should have no entries", 0, store.approximateNumEntries());
-
- store.put(0, "zero");
- store.put(1, "one");
- store.put(2, "two");
- store.put(4, "four");
- store.put(5, "five");
- assertEquals(5, store.approximateNumEntries());
- } finally {
- store.close();
- }
+ assertEquals("A newly created store should have no entries", 0, store.approximateNumEntries());
+
+ store.put(0, "zero");
+ store.put(1, "one");
+ store.put(2, "two");
+ store.put(4, "four");
+ store.put(5, "five");
+ assertEquals(5, store.approximateNumEntries());
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java
index 82071b7..fd9ea96 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreTest.java
@@ -19,7 +19,6 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.state.KeyValueStoreTestDriver;
import org.apache.kafka.streams.state.Stores;
import org.junit.Test;
@@ -51,54 +50,47 @@ public class InMemoryLRUCacheStoreTest extends AbstractKeyValueStoreTest {
@Test
public void testEvict() {
// Create the test driver ...
- KeyValueStoreTestDriver<Integer, String> driver = KeyValueStoreTestDriver.create(Integer.class, String.class);
- KeyValueStore<Integer, String> store = createKeyValueStore(driver.context(), Integer.class, String.class, false);
+ store.put(0, "zero");
+ store.put(1, "one");
+ store.put(2, "two");
+ store.put(3, "three");
+ store.put(4, "four");
+ store.put(5, "five");
+ store.put(6, "six");
+ store.put(7, "seven");
+ store.put(8, "eight");
+ store.put(9, "nine");
+ assertEquals(10, driver.sizeOf(store));
- try {
- store.put(0, "zero");
- store.put(1, "one");
- store.put(2, "two");
- store.put(3, "three");
- store.put(4, "four");
- store.put(5, "five");
- store.put(6, "six");
- store.put(7, "seven");
- store.put(8, "eight");
- store.put(9, "nine");
- assertEquals(10, driver.sizeOf(store));
+ store.put(10, "ten");
+ store.flush();
+ assertEquals(10, driver.sizeOf(store));
+ assertTrue(driver.flushedEntryRemoved(0));
+ assertEquals(1, driver.numFlushedEntryRemoved());
- store.put(10, "ten");
- store.flush();
- assertEquals(10, driver.sizeOf(store));
- assertTrue(driver.flushedEntryRemoved(0));
- assertEquals(1, driver.numFlushedEntryRemoved());
+ store.delete(1);
+ store.flush();
+ assertEquals(9, driver.sizeOf(store));
+ assertTrue(driver.flushedEntryRemoved(0));
+ assertTrue(driver.flushedEntryRemoved(1));
+ assertEquals(2, driver.numFlushedEntryRemoved());
- store.delete(1);
- store.flush();
- assertEquals(9, driver.sizeOf(store));
- assertTrue(driver.flushedEntryRemoved(0));
- assertTrue(driver.flushedEntryRemoved(1));
- assertEquals(2, driver.numFlushedEntryRemoved());
+ store.put(11, "eleven");
+ store.flush();
+ assertEquals(10, driver.sizeOf(store));
+ assertEquals(2, driver.numFlushedEntryRemoved());
- store.put(11, "eleven");
- store.flush();
- assertEquals(10, driver.sizeOf(store));
- assertEquals(2, driver.numFlushedEntryRemoved());
+ store.put(2, "two-again");
+ store.flush();
+ assertEquals(10, driver.sizeOf(store));
+ assertEquals(2, driver.numFlushedEntryRemoved());
- store.put(2, "two-again");
- store.flush();
- assertEquals(10, driver.sizeOf(store));
- assertEquals(2, driver.numFlushedEntryRemoved());
-
- store.put(12, "twelve");
- store.flush();
- assertEquals(10, driver.sizeOf(store));
- assertTrue(driver.flushedEntryRemoved(0));
- assertTrue(driver.flushedEntryRemoved(1));
- assertTrue(driver.flushedEntryRemoved(3));
- assertEquals(3, driver.numFlushedEntryRemoved());
- } finally {
- store.close();
- }
+ store.put(12, "twelve");
+ store.flush();
+ assertEquals(10, driver.sizeOf(store));
+ assertTrue(driver.flushedEntryRemoved(0));
+ assertTrue(driver.flushedEntryRemoved(1));
+ assertTrue(driver.flushedEntryRemoved(3));
+ assertEquals(3, driver.numFlushedEntryRemoved());
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
index 84c0320..521fa32 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
@@ -40,6 +40,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -64,7 +65,14 @@ public class RocksDBWindowStoreTest {
@SuppressWarnings("unchecked")
protected <K, V> WindowStore<K, V> createWindowStore(ProcessorContext context) {
- StateStoreSupplier supplier = new RocksDBWindowStoreSupplier<>(windowName, retentionPeriod, numSegments, true, intSerde, stringSerde);
+ StateStoreSupplier supplier = new RocksDBWindowStoreSupplier<>(windowName,
+ retentionPeriod,
+ numSegments,
+ true,
+ intSerde,
+ stringSerde,
+ true,
+ Collections.<String, String>emptyMap());
WindowStore<K, V> store = (WindowStore<K, V>) supplier.get();
store.init(context, store);
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/state/internals/StateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StateStoreTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StateStoreTestUtils.java
index ec5d841..39b127f 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StateStoreTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StateStoreTestUtils.java
@@ -29,12 +29,18 @@ import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
import org.apache.kafka.streams.state.StateSerdes;
import org.apache.kafka.test.MockProcessorContext;
+import java.util.Collections;
+
@SuppressWarnings("unchecked")
public class StateStoreTestUtils {
public static <K, V> KeyValueStore<K, V> newKeyValueStore(String name, Class<K> keyType, Class<V> valueType) {
final InMemoryKeyValueStoreSupplier<K, V> supplier = new InMemoryKeyValueStoreSupplier<>(name,
- null, null, new MockTime());
+ null,
+ null,
+ new MockTime(),
+ false,
+ Collections.<String, String>emptyMap());
final StateStore stateStore = supplier.get();
stateStore.init(new MockProcessorContext(StateSerdes.withBuiltinTypes(name, keyType, valueType),
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
index 19cd8e9..7675f9b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StoreChangeLoggerTest.java
@@ -25,7 +25,6 @@ import java.util.Map;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.StreamPartitioner;
import org.apache.kafka.streams.processor.internals.RecordCollector;
import org.apache.kafka.streams.state.StateSerdes;
@@ -41,7 +40,7 @@ public class StoreChangeLoggerTest {
private final Map<Integer, String> logged = new HashMap<>();
private final Map<Integer, String> written = new HashMap<>();
- private final ProcessorContext context = new MockProcessorContext(StateSerdes.withBuiltinTypes(topic, Integer.class, String.class),
+ private final MockProcessorContext context = new MockProcessorContext(StateSerdes.withBuiltinTypes(topic, Integer.class, String.class),
new RecordCollector(null, "StoreChangeLoggerTest") {
@SuppressWarnings("unchecked")
@Override
@@ -69,6 +68,7 @@ public class StoreChangeLoggerTest {
@Test
public void testAddRemove() {
+ context.setTime(1);
written.put(0, "zero");
changeLogger.add(0);
written.put(1, "one");
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/test/java/org/apache/kafka/test/MockStateStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/test/MockStateStoreSupplier.java b/streams/src/test/java/org/apache/kafka/test/MockStateStoreSupplier.java
index f24dfda..3532623 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockStateStoreSupplier.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockStateStoreSupplier.java
@@ -25,6 +25,8 @@ import org.apache.kafka.streams.processor.StateStore;
import org.apache.kafka.streams.processor.StateStoreSupplier;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map;
public class MockStateStoreSupplier implements StateStoreSupplier {
private final String name;
@@ -55,6 +57,16 @@ public class MockStateStoreSupplier implements StateStoreSupplier {
}
}
+ @Override
+ public Map<String, String> logConfig() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public boolean loggingEnabled() {
+ return loggingEnabled;
+ }
+
public static class MockStateStore implements StateStore {
private final String name;
private final boolean persistent;
[2/2] kafka git commit: KAFKA-3595: window stores use compact,
delete config for changelogs
Posted by gu...@apache.org.
KAFKA-3595: window stores use compact,delete config for changelogs
changelogs of window stores now configure cleanup.policy=compact,delete with retention.ms set to window maintainMs + StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG
StoreChangeLogger produces messages with context.timestamp().
Author: Damian Guy <da...@gmail.com>
Reviewers: Eno Thereska, Guozhang Wang
Closes #1792 from dguy/kafka-3595
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/69ebf6f7
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/69ebf6f7
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/69ebf6f7
Branch: refs/heads/trunk
Commit: 69ebf6f7be2fc0e471ebd5b7a166468017ff2651
Parents: eba0ede
Author: Damian Guy <da...@gmail.com>
Authored: Wed Sep 7 18:02:24 2016 -0700
Committer: Guozhang Wang <wa...@gmail.com>
Committed: Wed Sep 7 18:02:24 2016 -0700
----------------------------------------------------------------------
.../org/apache/kafka/streams/StreamsConfig.java | 12 +-
.../streams/kstream/internals/KTableImpl.java | 11 +-
.../kstream/internals/KTableStoreSupplier.java | 58 ----
.../streams/processor/StateStoreSupplier.java | 16 +
.../streams/processor/TimestampExtractor.java | 8 +-
.../streams/processor/TopologyBuilder.java | 71 +++--
.../internals/InternalTopicConfig.java | 110 +++++++
.../internals/InternalTopicManager.java | 65 ++--
.../internals/ProcessorContextImpl.java | 56 ++--
.../internals/StreamPartitionAssignor.java | 106 ++++---
.../streams/processor/internals/StreamTask.java | 7 +-
.../org/apache/kafka/streams/state/Stores.java | 87 +++++-
.../state/internals/AbstractStoreSupplier.java | 53 ++++
.../InMemoryKeyValueStoreSupplier.java | 24 +-
.../InMemoryLRUCacheStoreSupplier.java | 29 +-
.../internals/RocksDBKeyValueStoreSupplier.java | 29 +-
.../internals/RocksDBWindowStoreSupplier.java | 29 +-
.../state/internals/StoreChangeLogger.java | 2 +-
.../InternalTopicIntegrationTest.java | 110 +++++--
.../streams/processor/TopologyBuilderTest.java | 101 +++++-
.../internals/InternalTopicConfigTest.java | 122 ++++++++
.../internals/StreamPartitionAssignorTest.java | 8 +-
.../apache/kafka/streams/state/StoresTest.java | 84 +++++
.../internals/AbstractKeyValueStoreTest.java | 312 +++++++++----------
.../internals/InMemoryLRUCacheStoreTest.java | 82 +++--
.../state/internals/RocksDBWindowStoreTest.java | 10 +-
.../state/internals/StateStoreTestUtils.java | 8 +-
.../state/internals/StoreChangeLoggerTest.java | 4 +-
.../kafka/test/MockStateStoreSupplier.java | 12 +
29 files changed, 1094 insertions(+), 532 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index 41498cf..e972887 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -128,9 +128,13 @@ public class StreamsConfig extends AbstractConfig {
/** <code>client.id</code> */
public static final String CLIENT_ID_CONFIG = CommonClientConfigs.CLIENT_ID_CONFIG;
+ /** <code>rocksdb.config.setter</code> */
public static final String ROCKSDB_CONFIG_SETTER_CLASS_CONFIG = "rocksdb.config.setter";
public static final String ROCKSDB_CONFIG_SETTER_CLASS_DOC = "A Rocks DB config setter class that implements the <code>RocksDBConfigSetter</code> interface";
+ /** <code>windowstore.changelog.additional.retention.ms</code> */
+ public static final String WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG = "windowstore.changelog.additional.retention.ms";
+ public static final String WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_DOC = "Added to a windows maintainMs to ensure data is not deleted from the log prematurely. Allows for clock drift. Default is 1 day";
static {
@@ -238,7 +242,12 @@ public class StreamsConfig extends AbstractConfig {
Type.CLASS,
null,
Importance.LOW,
- ROCKSDB_CONFIG_SETTER_CLASS_DOC);
+ ROCKSDB_CONFIG_SETTER_CLASS_DOC)
+ .define(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG,
+ Type.LONG,
+ 24 * 60 * 60 * 1000,
+ Importance.MEDIUM,
+ WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_DOC);
}
// this is the list of configs for underlying clients
@@ -325,6 +334,7 @@ public class StreamsConfig extends AbstractConfig {
props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, getInt(REPLICATION_FACTOR_CONFIG));
props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, getInt(NUM_STANDBY_REPLICAS_CONFIG));
props.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, StreamPartitionAssignor.class.getName());
+ props.put(StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG, getLong(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG));
if (!getString(ZOOKEEPER_CONNECT_CONFIG).equals(""))
props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, getString(ZOOKEEPER_CONNECT_CONFIG));
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
index 2f36183..6c73b11 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java
@@ -33,10 +33,12 @@ import org.apache.kafka.streams.processor.ProcessorSupplier;
import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.processor.StreamPartitioner;
import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.state.internals.RocksDBKeyValueStoreSupplier;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
+import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@@ -416,10 +418,13 @@ public class KTableImpl<K, S, V> extends AbstractStream<K> implements KTable<K,
public void materialize(KTableSource<K, ?> source) {
synchronized (source) {
if (!source.isMaterialized()) {
- StateStoreSupplier storeSupplier =
- new KTableStoreSupplier<>(source.storeName, keySerde, valSerde, null);
+ StateStoreSupplier storeSupplier = new RocksDBKeyValueStoreSupplier<>(source.storeName,
+ keySerde,
+ valSerde,
+ false,
+ Collections.<String, String>emptyMap());
// mark this state as non internal hence it is read directly from a user topic
- topology.addStateStore(storeSupplier, false, name);
+ topology.addStateStore(storeSupplier, name);
source.materialize();
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableStoreSupplier.java
deleted file mode 100644
index ff118da..0000000
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableStoreSupplier.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kafka.streams.kstream.internals;
-
-import org.apache.kafka.common.serialization.Serde;
-import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.streams.processor.StateStore;
-import org.apache.kafka.streams.processor.StateStoreSupplier;
-import org.apache.kafka.streams.state.internals.MeteredKeyValueStore;
-import org.apache.kafka.streams.state.internals.RocksDBStore;
-
-/**
- * A KTable storage. It stores all entries in a local RocksDB database.
- *
- * @param <K> the type of keys
- * @param <V> the type of values
- */
-public class KTableStoreSupplier<K, V> implements StateStoreSupplier {
-
- private final String name;
- private final Serde<K> keySerde;
- private final Serde<V> valueSerde;
- private final Time time;
-
- protected KTableStoreSupplier(String name,
- Serde<K> keySerde,
- Serde<V> valueSerde,
- Time time) {
- this.name = name;
- this.keySerde = keySerde;
- this.valueSerde = valueSerde;
- this.time = time;
- }
-
- public String name() {
- return name;
- }
-
- public StateStore get() {
- return new MeteredKeyValueStore<>(new RocksDBStore<>(name, keySerde, valueSerde), "rocksdb-state", time);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/StateStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/StateStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/processor/StateStoreSupplier.java
index f2ae020..d3b0a1b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/StateStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/StateStoreSupplier.java
@@ -17,6 +17,8 @@
package org.apache.kafka.streams.processor;
+import java.util.Map;
+
/**
* A state store supplier which can create one or more {@link StateStore} instances.
*/
@@ -35,4 +37,18 @@ public interface StateStoreSupplier {
* @return a new {@link StateStore} instance
*/
StateStore get();
+
+ /**
+ * Returns a Map containing any log configs that will be used when creating the changelog for the {@link StateStore}
+ *
+ * Note: any unrecognized configs will be ignored by the Kafka brokers.
+ * @return Map containing any log configs to be used when creating the changelog for the {@link StateStore}
+ * If {@code loggingEnabled} returns false, this function will always return an empty map
+ */
+ Map<String, String> logConfig();
+
+ /**
+ * @return true if the {@link StateStore} should have logging enabled
+ */
+ boolean loggingEnabled();
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java b/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
index c872fa1..c55518b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/TimestampExtractor.java
@@ -18,6 +18,7 @@
package org.apache.kafka.streams.processor;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.streams.kstream.KTable;
/**
* An interface that allows the Kafka Streams framework to extract a timestamp from an instance of {@link ConsumerRecord}.
@@ -28,7 +29,12 @@ public interface TimestampExtractor {
/**
* Extracts a timestamp from a record.
* <p>
- * Typically, the timestamp represents the milliseconds since midnight, January 1, 1970 UTC.
+ * The extracted timestamp MUST represent the milliseconds since midnight, January 1, 1970 UTC.
+ *
+ * It is important to note that this timestamp may become the message timestamp for any messages sent to changelogs updated by {@link KTable}s
+ * and joins. The message timestamp is used for log retention and log rolling, so using nonsensical values may result in
+ * excessive log rolling and therefore broker performance degradation.
+ *
*
* @param record a data record
* @return the timestamp of the record
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
index bcdb54a..ee61e73 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
@@ -21,6 +21,7 @@ import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.Serializer;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.streams.errors.TopologyBuilderException;
+import org.apache.kafka.streams.processor.internals.InternalTopicConfig;
import org.apache.kafka.streams.processor.internals.ProcessorNode;
import org.apache.kafka.streams.processor.internals.ProcessorStateManager;
import org.apache.kafka.streams.processor.internals.ProcessorTopology;
@@ -28,6 +29,7 @@ import org.apache.kafka.streams.processor.internals.QuickUnion;
import org.apache.kafka.streams.processor.internals.SinkNode;
import org.apache.kafka.streams.processor.internals.SourceNode;
import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor.SubscriptionUpdates;
+import org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier;
import java.util.ArrayList;
import java.util.Arrays;
@@ -53,7 +55,6 @@ import java.util.regex.Pattern;
* instance that will then {@link org.apache.kafka.streams.KafkaStreams#start() begin consuming, processing, and producing records}.
*/
public class TopologyBuilder {
-
// node factories in a topological order
private final LinkedHashMap<String, NodeFactory> nodeFactories = new LinkedHashMap<>();
@@ -103,11 +104,9 @@ public class TopologyBuilder {
private static class StateStoreFactory {
public final Set<String> users;
- public final boolean isInternal;
public final StateStoreSupplier supplier;
- StateStoreFactory(boolean isInternal, StateStoreSupplier supplier) {
- this.isInternal = isInternal;
+ StateStoreFactory(StateStoreSupplier supplier) {
this.supplier = supplier;
this.users = new HashSet<>();
}
@@ -224,10 +223,10 @@ public class TopologyBuilder {
public static class TopicsInfo {
public Set<String> sinkTopics;
public Set<String> sourceTopics;
- public Set<String> interSourceTopics;
- public Set<String> stateChangelogTopics;
+ public Map<String, InternalTopicConfig> interSourceTopics;
+ public Map<String, InternalTopicConfig> stateChangelogTopics;
- public TopicsInfo(Set<String> sinkTopics, Set<String> sourceTopics, Set<String> interSourceTopics, Set<String> stateChangelogTopics) {
+ public TopicsInfo(Set<String> sinkTopics, Set<String> sourceTopics, Map<String, InternalTopicConfig> interSourceTopics, Map<String, InternalTopicConfig> stateChangelogTopics) {
this.sinkTopics = sinkTopics;
this.sourceTopics = sourceTopics;
this.interSourceTopics = interSourceTopics;
@@ -249,6 +248,16 @@ public class TopologyBuilder {
long n = ((long) sourceTopics.hashCode() << 32) | (long) stateChangelogTopics.hashCode();
return (int) (n % 0xFFFFFFFFL);
}
+
+ @Override
+ public String toString() {
+ return "TopicsInfo{" +
+ "sinkTopics=" + sinkTopics +
+ ", sourceTopics=" + sourceTopics +
+ ", interSourceTopics=" + interSourceTopics +
+ ", stateChangelogTopics=" + stateChangelogTopics +
+ '}';
+ }
}
/**
@@ -544,13 +553,13 @@ public class TopologyBuilder {
* @return this builder instance so methods can be chained together; never null
* @throws TopologyBuilderException if state store supplier is already added
*/
- public synchronized final TopologyBuilder addStateStore(StateStoreSupplier supplier, boolean isInternal, String... processorNames) {
+ public synchronized final TopologyBuilder addStateStore(StateStoreSupplier supplier, String... processorNames) {
Objects.requireNonNull(supplier, "supplier can't be null");
if (stateFactories.containsKey(supplier.name())) {
throw new TopologyBuilderException("StateStore " + supplier.name() + " is already added.");
}
- stateFactories.put(supplier.name(), new StateStoreFactory(isInternal, supplier));
+ stateFactories.put(supplier.name(), new StateStoreFactory(supplier));
if (processorNames != null) {
for (String processorName : processorNames) {
@@ -561,15 +570,6 @@ public class TopologyBuilder {
return this;
}
- /**
- * Adds a state store
- *
- * @param supplier the supplier used to obtain this state store {@link StateStore} instance
- * @return this builder instance so methods can be chained together; never null
- */
- public synchronized final TopologyBuilder addStateStore(StateStoreSupplier supplier, String... processorNames) {
- return this.addStateStore(supplier, true, processorNames);
- }
/**
* Connects the processor and the state stores
@@ -594,7 +594,6 @@ public class TopologyBuilder {
throw new TopologyBuilderException("Source store " + sourceStoreName + " is already added.");
}
sourceStoreToSourceTopic.put(sourceStoreName, topic);
-
return this;
}
@@ -842,8 +841,8 @@ public class TopologyBuilder {
for (Map.Entry<Integer, Set<String>> entry : nodeGroups.entrySet()) {
Set<String> sinkTopics = new HashSet<>();
Set<String> sourceTopics = new HashSet<>();
- Set<String> internalSourceTopics = new HashSet<>();
- Set<String> stateChangelogTopics = new HashSet<>();
+ Map<String, InternalTopicConfig> internalSourceTopics = new HashMap<>();
+ Map<String, InternalTopicConfig> stateChangelogTopics = new HashMap<>();
for (String node : entry.getValue()) {
// if the node is a source node, add to the source topics
String[] topics = nodeToSourceTopics.get(node);
@@ -853,7 +852,9 @@ public class TopologyBuilder {
if (this.internalTopicNames.contains(topic)) {
// prefix the internal topic name with the application id
String internalTopic = decorateTopic(topic);
- internalSourceTopics.add(internalTopic);
+ internalSourceTopics.put(internalTopic, new InternalTopicConfig(internalTopic,
+ Collections.singleton(InternalTopicConfig.CleanupPolicy.delete),
+ Collections.<String, String>emptyMap()));
sourceTopics.add(internalTopic);
} else {
sourceTopics.add(topic);
@@ -874,22 +875,38 @@ public class TopologyBuilder {
// if the node is connected to a state, add to the state topics
for (StateStoreFactory stateFactory : stateFactories.values()) {
- if (stateFactory.isInternal && stateFactory.users.contains(node)) {
- // prefix the change log topic name with the application id
- stateChangelogTopics.add(ProcessorStateManager.storeChangelogTopic(applicationId, stateFactory.supplier.name()));
+ final StateStoreSupplier supplier = stateFactory.supplier;
+ if (supplier.loggingEnabled() && stateFactory.users.contains(node)) {
+ final String name = ProcessorStateManager.storeChangelogTopic(applicationId, supplier.name());
+ final InternalTopicConfig internalTopicConfig = createInternalTopicConfig(supplier, name);
+ stateChangelogTopics.put(name, internalTopicConfig);
}
}
}
topicGroups.put(entry.getKey(), new TopicsInfo(
Collections.unmodifiableSet(sinkTopics),
Collections.unmodifiableSet(sourceTopics),
- Collections.unmodifiableSet(internalSourceTopics),
- Collections.unmodifiableSet(stateChangelogTopics)));
+ Collections.unmodifiableMap(internalSourceTopics),
+ Collections.unmodifiableMap(stateChangelogTopics)));
}
return Collections.unmodifiableMap(topicGroups);
}
+ private InternalTopicConfig createInternalTopicConfig(final StateStoreSupplier supplier, final String name) {
+ if (!(supplier instanceof RocksDBWindowStoreSupplier)) {
+ return new InternalTopicConfig(name, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), supplier.logConfig());
+ }
+
+ final RocksDBWindowStoreSupplier windowStoreSupplier = (RocksDBWindowStoreSupplier) supplier;
+ final InternalTopicConfig config = new InternalTopicConfig(name,
+ Utils.mkSet(InternalTopicConfig.CleanupPolicy.compact,
+ InternalTopicConfig.CleanupPolicy.delete),
+ supplier.logConfig());
+ config.setRetentionMs(windowStoreSupplier.retentionPeriod());
+ return config;
+ }
+
/**
* Get the names of topics that are to be consumed by the source nodes created by this builder.
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
new file mode 100644
index 0000000..45016c8
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * InternalTopicConfig captures the properties required for configuring
+ * the internal topics we create for change-logs and repartitioning etc.
+ */
+public class InternalTopicConfig {
+ public enum CleanupPolicy { compact, delete }
+
+ private final String name;
+ private final Map<String, String> logConfig;
+ private Long retentionMs;
+ private final Set<CleanupPolicy> cleanupPolicies;
+
+ public InternalTopicConfig(final String name, final Set<CleanupPolicy> defaultCleanupPolicies, final Map<String, String> logConfig) {
+ Objects.requireNonNull(name, "name can't be null");
+ if (defaultCleanupPolicies.isEmpty()) {
+ throw new IllegalArgumentException("Must provide at least one cleanup policy");
+ }
+ this.name = name;
+ this.cleanupPolicies = defaultCleanupPolicies;
+ this.logConfig = logConfig;
+ }
+
+ /* for test use only */
+ boolean isCompacted() {
+ return cleanupPolicies.contains(CleanupPolicy.compact);
+ }
+
+ private boolean isCompactDelete() {
+ return cleanupPolicies.contains(CleanupPolicy.compact) && cleanupPolicies.contains(CleanupPolicy.delete);
+ }
+
+ /**
+ * Get the configured properties for this topic. If rententionMs is set then
+ * we add additionalRetentionMs to work out the desired retention when cleanup.policy=compact,delete
+ *
+ * @param additionalRetentionMs - added to retention to allow for clock drift etc
+ * @return Properties to be used when creating the topic
+ */
+ public Properties toProperties(final long additionalRetentionMs) {
+ final Properties result = new Properties();
+ for (Map.Entry<String, String> configEntry : logConfig.entrySet()) {
+ result.put(configEntry.getKey(), configEntry.getValue());
+ }
+ if (retentionMs != null && isCompactDelete()) {
+ result.put(InternalTopicManager.RETENTION_MS, String.valueOf(retentionMs + additionalRetentionMs));
+ }
+
+ if (!logConfig.containsKey(InternalTopicManager.CLEANUP_POLICY_PROP)) {
+ final StringBuilder builder = new StringBuilder();
+ for (CleanupPolicy cleanupPolicy : cleanupPolicies) {
+ builder.append(cleanupPolicy.name()).append(",");
+ }
+ builder.deleteCharAt(builder.length() - 1);
+
+ result.put(InternalTopicManager.CLEANUP_POLICY_PROP, builder.toString());
+ }
+
+
+ return result;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public void setRetentionMs(final long retentionMs) {
+ if (!logConfig.containsKey(InternalTopicManager.RETENTION_MS)) {
+ this.retentionMs = retentionMs;
+ }
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ final InternalTopicConfig that = (InternalTopicConfig) o;
+ return Objects.equals(name, that.name) &&
+ Objects.equals(logConfig, that.logConfig) &&
+ Objects.equals(retentionMs, that.retentionMs) &&
+ Objects.equals(cleanupPolicies, that.cleanupPolicies);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, logConfig, retentionMs, cleanupPolicies);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
index 4477fb7..44de757 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicManager.java
@@ -24,6 +24,7 @@ import org.I0Itec.zkclient.ZkClient;
import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
import org.I0Itec.zkclient.serialize.ZkSerializer;
+import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.zookeeper.ZooDefs;
import org.slf4j.Logger;
@@ -35,8 +36,10 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
-import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
public class InternalTopicManager {
@@ -48,11 +51,27 @@ public class InternalTopicManager {
private static final String ZK_DELETE_TOPIC_PATH = "/admin/delete_topics";
private static final String ZK_ENTITY_CONFIG_PATH = "/config/topics";
// TODO: the following LogConfig dependency should be removed after KIP-4
- private static final String CLEANUP_POLICY_PROP = "cleanup.policy";
- private static final String COMPACT = "compact";
+ public static final String CLEANUP_POLICY_PROP = "cleanup.policy";
+ private static final Set<String> CLEANUP_POLICIES = Utils.mkSet("compact", "delete");
+ public static final String RETENTION_MS = "retention.ms";
+ public static final Long WINDOW_CHANGE_LOG_ADDITIONAL_RETENTION_DEFAULT = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS);
private final ZkClient zkClient;
private final int replicationFactor;
+ private final long windowChangeLogAdditionalRetention;
+
+ public static boolean isValidCleanupPolicy(final String cleanupPolicy) {
+ if (cleanupPolicy == null) {
+ return false;
+ }
+ final String[] policies = cleanupPolicy.toLowerCase(Locale.ROOT).split(",");
+ for (String policy : policies) {
+ if (!CLEANUP_POLICIES.contains(policy.trim())) {
+ return false;
+ }
+ }
+ return true;
+ }
private class ZKStringSerializer implements ZkSerializer {
@@ -87,22 +106,24 @@ public class InternalTopicManager {
public InternalTopicManager() {
this.zkClient = null;
this.replicationFactor = 0;
+ this.windowChangeLogAdditionalRetention = WINDOW_CHANGE_LOG_ADDITIONAL_RETENTION_DEFAULT;
}
- public InternalTopicManager(String zkConnect, int replicationFactor) {
+ public InternalTopicManager(String zkConnect, final int replicationFactor, long windowChangeLogAdditionalRetention) {
this.zkClient = new ZkClient(zkConnect, 30 * 1000, 30 * 1000, new ZKStringSerializer());
this.replicationFactor = replicationFactor;
+ this.windowChangeLogAdditionalRetention = windowChangeLogAdditionalRetention;
}
- public void makeReady(String topic, int numPartitions, boolean compactTopic) {
+ public void makeReady(InternalTopicConfig topic, int numPartitions) {
boolean topicNotReady = true;
while (topicNotReady) {
- Map<Integer, List<Integer>> topicMetadata = getTopicMetadata(topic);
+ Map<Integer, List<Integer>> topicMetadata = getTopicMetadata(topic.name());
if (topicMetadata == null) {
try {
- createTopic(topic, numPartitions, replicationFactor, compactTopic);
+ createTopic(topic, numPartitions, replicationFactor);
} catch (ZkNodeExistsException e) {
// ignore and continue
}
@@ -110,14 +131,14 @@ public class InternalTopicManager {
if (topicMetadata.size() > numPartitions) {
// else if topic exists with more #.partitions than needed, delete in order to re-create it
try {
- deleteTopic(topic);
+ deleteTopic(topic.name());
} catch (ZkNodeExistsException e) {
// ignore and continue
}
} else if (topicMetadata.size() < numPartitions) {
// else if topic exists with less #.partitions than needed, add partitions
try {
- addPartitions(topic, numPartitions - topicMetadata.size(), replicationFactor, topicMetadata);
+ addPartitions(topic.name(), numPartitions - topicMetadata.size(), replicationFactor, topicMetadata);
} catch (ZkNoNodeException e) {
// ignore and continue
}
@@ -163,9 +184,8 @@ public class InternalTopicManager {
}
}
- private void createTopic(String topic, int numPartitions, int replicationFactor, boolean compactTopic) throws ZkNodeExistsException {
- log.debug("Creating topic {} with {} partitions from ZK in partition assignor.", topic, numPartitions);
- Properties prop = new Properties();
+ private void createTopic(InternalTopicConfig topic, int numPartitions, int replicationFactor) throws ZkNodeExistsException {
+ log.debug("Creating topic {} with {} partitions from ZK in partition assignor.", topic.name(), numPartitions);
ObjectMapper mapper = new ObjectMapper();
List<Integer> brokers = getBrokers();
int numBrokers = brokers.size();
@@ -185,17 +205,14 @@ public class InternalTopicManager {
assignment.put(i, brokerList);
}
// write out config first just like in AdminUtils.scala createOrUpdateTopicPartitionAssignmentPathInZK()
- if (compactTopic) {
- prop.put(CLEANUP_POLICY_PROP, COMPACT);
- try {
- Map<String, Object> dataMap = new HashMap<>();
- dataMap.put("version", 1);
- dataMap.put("config", prop);
- String data = mapper.writeValueAsString(dataMap);
- zkClient.createPersistent(ZK_ENTITY_CONFIG_PATH + "/" + topic, data, ZooDefs.Ids.OPEN_ACL_UNSAFE);
- } catch (JsonProcessingException e) {
- throw new StreamsException("Error while creating topic config in ZK for internal topic " + topic, e);
- }
+ try {
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("version", 1);
+ dataMap.put("config", topic.toProperties(windowChangeLogAdditionalRetention));
+ String data = mapper.writeValueAsString(dataMap);
+ zkClient.createPersistent(ZK_ENTITY_CONFIG_PATH + "/" + topic.name(), data, ZooDefs.Ids.OPEN_ACL_UNSAFE);
+ } catch (JsonProcessingException e) {
+ throw new StreamsException("Error while creating topic config in ZK for internal topic " + topic, e);
}
// try to write to ZK with open ACL
@@ -205,7 +222,7 @@ public class InternalTopicManager {
dataMap.put("partitions", assignment);
String data = mapper.writeValueAsString(dataMap);
- zkClient.createPersistent(ZK_TOPIC_PATH + "/" + topic, data, ZooDefs.Ids.OPEN_ACL_UNSAFE);
+ zkClient.createPersistent(ZK_TOPIC_PATH + "/" + topic.name(), data, ZooDefs.Ids.OPEN_ACL_UNSAFE);
} catch (JsonProcessingException e) {
throw new StreamsException("Error while creating topic metadata in ZK for internal topic " + topic, e);
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
index 00ffb20..a38839f 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextImpl.java
@@ -44,6 +44,10 @@ public class ProcessorContextImpl implements ProcessorContext, RecordCollector.S
private final Serde<?> valSerde;
private boolean initialized;
+ private Long timestamp;
+ private String topic;
+ private Long offset;
+ private Integer partition;
@SuppressWarnings("unchecked")
public ProcessorContextImpl(TaskId id,
@@ -136,53 +140,46 @@ public class ProcessorContextImpl implements ProcessorContext, RecordCollector.S
return stateMgr.getStore(name);
}
- /**
- * @throws IllegalStateException if the task's record is null
- */
- @Override
- public String topic() {
- if (task.record() == null)
- throw new IllegalStateException("This should not happen as topic() should only be called while a record is processed");
- String topic = task.record().topic();
-
- if (topic.equals(NONEXIST_TOPIC))
+ @Override
+ public synchronized String topic() {
+ if (topic == null || topic.equals(NONEXIST_TOPIC))
return null;
else
return topic;
}
/**
- * @throws IllegalStateException if the task's record is null
+ * @throws IllegalStateException if partition is null
*/
@Override
- public int partition() {
- if (task.record() == null)
+ public synchronized int partition() {
+ if (partition == null) {
throw new IllegalStateException("This should not happen as partition() should only be called while a record is processed");
-
- return task.record().partition();
+ }
+ return partition;
}
/**
- * @throws IllegalStateException if the task's record is null
+ * @throws IllegalStateException if offset is null
*/
@Override
- public long offset() {
- if (this.task.record() == null)
+ public synchronized long offset() {
+ if (offset == null) {
throw new IllegalStateException("This should not happen as offset() should only be called while a record is processed");
-
- return this.task.record().offset();
+ }
+ return offset;
}
/**
- * @throws IllegalStateException if the task's record is null
+ * @throws IllegalStateException if timestamp is null
*/
@Override
- public long timestamp() {
- if (task.record() == null)
- throw new IllegalStateException("This should not happen as timestamp() should only be called while a record is processed");
-
- return task.record().timestamp;
+ public synchronized long timestamp() {
+ if (timestamp == null) {
+ throw new IllegalStateException("This should not happen as timestamp should be set during record processing");
+ }
+ return timestamp;
}
@Override
@@ -219,4 +216,11 @@ public class ProcessorContextImpl implements ProcessorContext, RecordCollector.S
public Map<String, Object> appConfigsWithPrefix(String prefix) {
return config.originalsWithPrefix(prefix);
}
+
+ public synchronized void update(final StampedRecord record) {
+ this.timestamp = record.timestamp;
+ this.partition = record.partition();
+ this.offset = record.offset();
+ this.topic = record.topic();
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
index bb8379c..65eda80 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamPartitionAssignor.java
@@ -52,6 +52,8 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import static org.apache.kafka.streams.processor.internals.InternalTopicManager.WINDOW_CHANGE_LOG_ADDITIONAL_RETENTION_DEFAULT;
+
public class StreamPartitionAssignor implements PartitionAssignor, Configurable {
private static final Logger log = LoggerFactory.getLogger(StreamPartitionAssignor.class);
@@ -93,8 +95,8 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
private int numStandbyReplicas;
private Map<Integer, TopologyBuilder.TopicsInfo> topicGroups;
private Map<TopicPartition, Set<TaskId>> partitionToTaskIds;
- private Map<String, Set<TaskId>> stateChangelogTopicToTaskIds;
- private Map<String, Set<TaskId>> internalSourceTopicToTaskIds;
+ private Map<InternalTopicConfig, Set<TaskId>> stateChangelogTopicToTaskIds;
+ private Map<InternalTopicConfig, Set<TaskId>> internalSourceTopicToTaskIds;
private Map<TaskId, Set<TopicPartition>> standbyTasks;
private InternalTopicManager internalTopicManager;
@@ -109,6 +111,7 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
public void configure(Map<String, ?> configs) {
numStandbyReplicas = (Integer) configs.get(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG);
+
Object o = configs.get(StreamsConfig.InternalConfig.STREAM_THREAD_INSTANCE);
if (o == null) {
KafkaException ex = new KafkaException("StreamThread is not specified");
@@ -147,7 +150,10 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
if (configs.containsKey(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG)) {
internalTopicManager = new InternalTopicManager(
(String) configs.get(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG),
- configs.containsKey(StreamsConfig.REPLICATION_FACTOR_CONFIG) ? (Integer) configs.get(StreamsConfig.REPLICATION_FACTOR_CONFIG) : 1);
+ configs.containsKey(StreamsConfig.REPLICATION_FACTOR_CONFIG) ? (Integer) configs.get(StreamsConfig.REPLICATION_FACTOR_CONFIG) : 1,
+ configs.containsKey(StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG) ?
+ (Long) configs.get(StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG)
+ : WINDOW_CHANGE_LOG_ADDITIONAL_RETENTION_DEFAULT);
} else {
log.info("stream-thread [{}] Config '{}' isn't supplied and hence no internal topics will be created.", streamThread.getName(), StreamsConfig.ZOOKEEPER_CONNECT_CONFIG);
}
@@ -176,23 +182,20 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
/**
* Internal helper function that creates a Kafka topic
* @param topicToTaskIds Map that contains the topic names to be created
- * @param compactTopic If true, the topic should be a compacted topic. This is used for
- * change log topics usually.
* @param postPartitionPhase If true, the computation for calculating the number of partitions
* is slightly different. Set to true after the initial topic-to-partition
* assignment.
* @return
*/
- private Map<TopicPartition, PartitionInfo> prepareTopic(Map<String, Set<TaskId>> topicToTaskIds,
- boolean compactTopic,
+ private Map<TopicPartition, PartitionInfo> prepareTopic(Map<InternalTopicConfig, Set<TaskId>> topicToTaskIds,
boolean postPartitionPhase) {
Map<TopicPartition, PartitionInfo> partitionInfos = new HashMap<>();
// if ZK is specified, prepare the internal source topic before calling partition grouper
if (internalTopicManager != null) {
log.debug("stream-thread [{}] Starting to validate internal topics in partition assignor.", streamThread.getName());
- for (Map.Entry<String, Set<TaskId>> entry : topicToTaskIds.entrySet()) {
- String topic = entry.getKey();
+ for (Map.Entry<InternalTopicConfig, Set<TaskId>> entry : topicToTaskIds.entrySet()) {
+ InternalTopicConfig topic = entry.getKey();
int numPartitions = 0;
if (postPartitionPhase) {
// the expected number of partitions is the max value of TaskId.partition + 1
@@ -208,12 +211,12 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
}
}
- internalTopicManager.makeReady(topic, numPartitions, compactTopic);
+ internalTopicManager.makeReady(topic, numPartitions);
// wait until the topic metadata has been propagated to all brokers
List<PartitionInfo> partitions;
do {
- partitions = streamThread.restoreConsumer.partitionsFor(topic);
+ partitions = streamThread.restoreConsumer.partitionsFor(topic.name());
} while (partitions == null || partitions.size() != numPartitions);
for (PartitionInfo partition : partitions)
@@ -223,10 +226,10 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
log.info("stream-thread [{}] Completed validating internal topics in partition assignor", streamThread.getName());
} else {
List<String> missingTopics = new ArrayList<>();
- for (String topic : topicToTaskIds.keySet()) {
- List<PartitionInfo> partitions = streamThread.restoreConsumer.partitionsFor(topic);
+ for (InternalTopicConfig topic : topicToTaskIds.keySet()) {
+ List<PartitionInfo> partitions = streamThread.restoreConsumer.partitionsFor(topic.name());
if (partitions == null) {
- missingTopics.add(topic);
+ missingTopics.add(topic.name());
}
}
if (!missingTopics.isEmpty()) {
@@ -295,21 +298,21 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
// and enforce the number of partitions for those internal topics.
internalSourceTopicToTaskIds = new HashMap<>();
Map<Integer, Set<String>> sourceTopicGroups = new HashMap<>();
- Map<Integer, Set<String>> internalSourceTopicGroups = new HashMap<>();
+ Map<Integer, Collection<InternalTopicConfig>> internalSourceTopicGroups = new HashMap<>();
for (Map.Entry<Integer, TopologyBuilder.TopicsInfo> entry : topicGroups.entrySet()) {
sourceTopicGroups.put(entry.getKey(), entry.getValue().sourceTopics);
- internalSourceTopicGroups.put(entry.getKey(), entry.getValue().interSourceTopics);
+ internalSourceTopicGroups.put(entry.getKey(), entry.getValue().interSourceTopics.values());
}
// for all internal source topics
// set the number of partitions to the maximum of the depending sub-topologies source topics
Map<TopicPartition, PartitionInfo> internalPartitionInfos = new HashMap<>();
- Set<String> allInternalTopicNames = new HashSet<>();
+ Map<String, InternalTopicConfig> allInternalTopics = new HashMap<>();
for (Map.Entry<Integer, TopologyBuilder.TopicsInfo> entry : topicGroups.entrySet()) {
- Set<String> internalTopics = entry.getValue().interSourceTopics;
- allInternalTopicNames.addAll(internalTopics);
- for (String internalTopic : internalTopics) {
+ Map<String, InternalTopicConfig> internalTopics = entry.getValue().interSourceTopics;
+ allInternalTopics.putAll(internalTopics);
+ for (InternalTopicConfig internalTopic : internalTopics.values()) {
Set<TaskId> tasks = internalSourceTopicToTaskIds.get(internalTopic);
if (tasks == null) {
@@ -317,13 +320,13 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
for (Map.Entry<Integer, TopologyBuilder.TopicsInfo> other : topicGroups.entrySet()) {
Set<String> otherSinkTopics = other.getValue().sinkTopics;
- if (otherSinkTopics.contains(internalTopic)) {
+ if (otherSinkTopics.contains(internalTopic.name())) {
for (String topic : other.getValue().sourceTopics) {
Integer partitions = null;
// It is possible the sourceTopic is another internal topic, i.e,
// map().join().join(map())
- if (allInternalTopicNames.contains(topic)) {
- Set<TaskId> taskIds = internalSourceTopicToTaskIds.get(topic);
+ if (allInternalTopics.containsKey(topic)) {
+ Set<TaskId> taskIds = internalSourceTopicToTaskIds.get(allInternalTopics.get(topic));
if (taskIds != null) {
for (TaskId taskId : taskIds) {
partitions = taskId.partition;
@@ -340,8 +343,8 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
}
internalSourceTopicToTaskIds.put(internalTopic, Collections.singleton(new TaskId(entry.getKey(), numPartitions)));
for (int partition = 0; partition < numPartitions; partition++) {
- internalPartitionInfos.put(new TopicPartition(internalTopic, partition),
- new PartitionInfo(internalTopic, partition, null, new Node[0], new Node[0]));
+ internalPartitionInfos.put(new TopicPartition(internalTopic.name(), partition),
+ new PartitionInfo(internalTopic.name(), partition, null, new Node[0], new Node[0]));
}
}
}
@@ -353,7 +356,7 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
metadata.withPartitions(internalPartitionInfos));
- internalPartitionInfos = prepareTopic(internalSourceTopicToTaskIds, false, false);
+ internalPartitionInfos = prepareTopic(internalSourceTopicToTaskIds, false);
internalSourceTopicToTaskIds.clear();
metadataWithInternalTopics = metadata;
@@ -367,21 +370,23 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
// add tasks to state change log topic subscribers
stateChangelogTopicToTaskIds = new HashMap<>();
for (TaskId task : partitionsForTask.keySet()) {
- for (String topicName : topicGroups.get(task.topicGroupId).stateChangelogTopics) {
- Set<TaskId> tasks = stateChangelogTopicToTaskIds.get(topicName);
+ final Map<String, InternalTopicConfig> stateChangelogTopics = topicGroups.get(task.topicGroupId).stateChangelogTopics;
+ for (InternalTopicConfig topic : stateChangelogTopics.values()) {
+ Set<TaskId> tasks = stateChangelogTopicToTaskIds.get(topic);
if (tasks == null) {
tasks = new HashSet<>();
- stateChangelogTopicToTaskIds.put(topicName, tasks);
+ stateChangelogTopicToTaskIds.put(topic, tasks);
}
tasks.add(task);
}
- for (String topicName : topicGroups.get(task.topicGroupId).interSourceTopics) {
- Set<TaskId> tasks = internalSourceTopicToTaskIds.get(topicName);
+ final Map<String, InternalTopicConfig> interSourceTopics = topicGroups.get(task.topicGroupId).interSourceTopics;
+ for (InternalTopicConfig topic : interSourceTopics.values()) {
+ Set<TaskId> tasks = internalSourceTopicToTaskIds.get(topic);
if (tasks == null) {
tasks = new HashSet<>();
- internalSourceTopicToTaskIds.put(topicName, tasks);
+ internalSourceTopicToTaskIds.put(topic, tasks);
}
tasks.add(task);
@@ -461,9 +466,9 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
}
// if ZK is specified, validate the internal topics again
- prepareTopic(internalSourceTopicToTaskIds, false /* compactTopic */, true);
+ prepareTopic(internalSourceTopicToTaskIds, /* compactTopic */ true);
// change log topics should be compacted
- prepareTopic(stateChangelogTopicToTaskIds, true /* compactTopic */, true);
+ prepareTopic(stateChangelogTopicToTaskIds, /* compactTopic */ true);
Map<String, Assignment> assignment = new HashMap<>();
for (AssignmentSupplier assignmentSupplier : assignmentSuppliers) {
@@ -563,21 +568,24 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
return metadataWithInternalTopics;
}
- private void ensureCopartitioning(Collection<Set<String>> copartitionGroups, Map<Integer, Set<String>> internalTopicGroups, Cluster metadata) {
- Set<String> internalTopics = new HashSet<>();
- for (Set<String> topics : internalTopicGroups.values())
- internalTopics.addAll(topics);
+ private void ensureCopartitioning(Collection<Set<String>> copartitionGroups, Map<Integer, Collection<InternalTopicConfig>> internalTopicGroups, Cluster metadata) {
+ Map<String, InternalTopicConfig> internalTopics = new HashMap<>();
+ for (Collection<InternalTopicConfig> topics : internalTopicGroups.values()) {
+ for (InternalTopicConfig topic : topics) {
+ internalTopics.put(topic.name(), topic);
+ }
+ }
for (Set<String> copartitionGroup : copartitionGroups) {
ensureCopartitioning(copartitionGroup, internalTopics, metadata);
}
}
- private void ensureCopartitioning(Set<String> copartitionGroup, Set<String> internalTopics, Cluster metadata) {
+ private void ensureCopartitioning(Set<String> copartitionGroup, Map<String, InternalTopicConfig> internalTopics, Cluster metadata) {
int numPartitions = -1;
for (String topic : copartitionGroup) {
- if (!internalTopics.contains(topic)) {
+ if (!internalTopics.containsKey(topic)) {
List<PartitionInfo> infos = metadata.partitionsForTopic(topic);
if (infos == null)
@@ -594,9 +602,9 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
}
if (numPartitions == -1) {
- for (String topic : internalTopics) {
- if (copartitionGroup.contains(topic)) {
- Integer partitions = metadata.partitionCountForTopic(topic);
+ for (InternalTopicConfig topic : internalTopics.values()) {
+ if (copartitionGroup.contains(topic.name())) {
+ Integer partitions = metadata.partitionCountForTopic(topic.name());
if (partitions != null && partitions > numPartitions) {
numPartitions = partitions;
}
@@ -604,8 +612,8 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
}
}
// enforce co-partitioning restrictions to internal topics reusing internalSourceTopicToTaskIds
- for (String topic : internalTopics) {
- if (copartitionGroup.contains(topic)) {
+ for (InternalTopicConfig topic : internalTopics.values()) {
+ if (copartitionGroup.contains(topic.name())) {
internalSourceTopicToTaskIds
.put(topic, Collections.singleton(new TaskId(-1, numPartitions)));
}
@@ -614,7 +622,13 @@ public class StreamPartitionAssignor implements PartitionAssignor, Configurable
/* For Test Only */
public Set<TaskId> tasksForState(String stateName) {
- return stateChangelogTopicToTaskIds.get(ProcessorStateManager.storeChangelogTopic(streamThread.applicationId, stateName));
+ final String changeLogName = ProcessorStateManager.storeChangelogTopic(streamThread.applicationId, stateName);
+ for (InternalTopicConfig internalTopicConfig : stateChangelogTopicToTaskIds.keySet()) {
+ if (internalTopicConfig.name().equals(changeLogName)) {
+ return stateChangelogTopicToTaskIds.get(internalTopicConfig);
+ }
+ }
+ return Collections.emptySet();
}
public Set<TaskId> tasksForPartition(TopicPartition partition) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index 18ca0ee..476ec2e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -171,6 +171,7 @@ public class StreamTask extends AbstractTask implements Punctuator {
log.debug("task [{}] Start processing one record [{}]", id(), currRecord);
+ updateContext(currRecord);
this.currNode.process(currRecord.key(), currRecord.value());
log.debug("task [{}] Completed processing one record [{}]", id(), currRecord);
@@ -226,7 +227,7 @@ public class StreamTask extends AbstractTask implements Punctuator {
currNode = node;
currRecord = new StampedRecord(DUMMY_RECORD, timestamp);
-
+ updateContext(currRecord);
try {
node.processor().punctuate(timestamp);
} finally {
@@ -235,6 +236,10 @@ public class StreamTask extends AbstractTask implements Punctuator {
}
}
+ private void updateContext(final StampedRecord record) {
+ ((ProcessorContextImpl) processorContext).update(record);
+ }
+
public StampedRecord record() {
return this.currRecord;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
index 9f1e53c..03c0d02 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
@@ -25,6 +25,8 @@ import org.apache.kafka.streams.state.internals.RocksDBKeyValueStoreSupplier;
import org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
/**
* Factory for creating state stores in Kafka Streams.
@@ -44,11 +46,15 @@ public class Stores {
return new ValueFactory<K>() {
@Override
public <V> KeyValueFactory<K, V> withValues(final Serde<V> valueSerde) {
+
return new KeyValueFactory<K, V>() {
+
@Override
public InMemoryKeyValueFactory<K, V> inMemory() {
return new InMemoryKeyValueFactory<K, V>() {
private int capacity = Integer.MAX_VALUE;
+ private final Map<String, String> logConfig = new HashMap<>();
+ private boolean logged = true;
/**
* @param capacity the maximum capacity of the in-memory cache; should be one less than a power of 2
@@ -62,11 +68,25 @@ public class Stores {
}
@Override
+ public InMemoryKeyValueFactory<K, V> enableLogging(final Map<String, String> config) {
+ logged = true;
+ logConfig.putAll(config);
+ return this;
+ }
+
+ @Override
+ public InMemoryKeyValueFactory<K, V> disableLogging() {
+ logged = false;
+ logConfig.clear();
+ return this;
+ }
+
+ @Override
public StateStoreSupplier build() {
if (capacity < Integer.MAX_VALUE) {
- return new InMemoryLRUCacheStoreSupplier<>(name, capacity, keySerde, valueSerde);
+ return new InMemoryLRUCacheStoreSupplier<>(name, capacity, keySerde, valueSerde, logged, logConfig);
}
- return new InMemoryKeyValueStoreSupplier<>(name, keySerde, valueSerde);
+ return new InMemoryKeyValueStoreSupplier<>(name, keySerde, valueSerde, logged, logConfig);
}
};
}
@@ -74,9 +94,11 @@ public class Stores {
@Override
public PersistentKeyValueFactory<K, V> persistent() {
return new PersistentKeyValueFactory<K, V>() {
+ private final Map<String, String> logConfig = new HashMap<>();
private int numSegments = 0;
private long retentionPeriod = 0L;
private boolean retainDuplicates = false;
+ private boolean logged = true;
@Override
public PersistentKeyValueFactory<K, V> windowed(long retentionPeriod, int numSegments, boolean retainDuplicates) {
@@ -88,15 +110,31 @@ public class Stores {
}
@Override
+ public PersistentKeyValueFactory<K, V> enableLogging(final Map<String, String> config) {
+ logged = true;
+ logConfig.putAll(config);
+ return this;
+ }
+
+ @Override
+ public PersistentKeyValueFactory<K, V> disableLogging() {
+ logged = false;
+ logConfig.clear();
+ return this;
+ }
+
+ @Override
public StateStoreSupplier build() {
if (numSegments > 0) {
- return new RocksDBWindowStoreSupplier<>(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde);
+ return new RocksDBWindowStoreSupplier<>(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde, logged, logConfig);
}
- return new RocksDBKeyValueStoreSupplier<>(name, keySerde, valueSerde);
+ return new RocksDBKeyValueStoreSupplier<>(name, keySerde, valueSerde, logged, logConfig);
}
};
}
+
+
};
}
};
@@ -104,6 +142,7 @@ public class Stores {
};
}
+
public static abstract class StoreFactory {
/**
* Begin to create a {@link KeyValueStore} by specifying the keys will be {@link String}s.
@@ -257,12 +296,7 @@ public class Stores {
public abstract <V> KeyValueFactory<K, V> withValues(Serde<V> valueSerde);
}
- /**
- * The interface used to specify the different kinds of key-value stores.
- *
- * @param <K> the type of keys
- * @param <V> the type of values
- */
+
public interface KeyValueFactory<K, V> {
/**
* Keep all key-value entries in-memory, although for durability all entries are recorded in a Kafka topic that can be
@@ -299,6 +333,23 @@ public class Stores {
InMemoryKeyValueFactory<K, V> maxEntries(int capacity);
/**
+ * Indicates that a changelog should be created for the store. The changelog will be created
+ * with the provided cleanupPolicy and configs.
+ *
+ * Note: Any unrecognized configs will be ignored.
+ * @param config any configs that should be applied to the changelog
+ * @return the factory to create an in-memory key-value store
+ */
+ InMemoryKeyValueFactory<K, V> enableLogging(final Map<String, String> config);
+
+ /**
+ * Indicates that a changelog should not be created for the key-value store
+ * @return the factory to create an in-memory key-value store
+ */
+ InMemoryKeyValueFactory<K, V> disableLogging();
+
+
+ /**
* Return the instance of StateStoreSupplier of new key-value store.
* @return the state store supplier; never null
*/
@@ -323,6 +374,22 @@ public class Stores {
PersistentKeyValueFactory<K, V> windowed(long retentionPeriod, int numSegments, boolean retainDuplicates);
/**
+ * Indicates that a changelog should be created for the store. The changelog will be created
+ * with the provided cleanupPolicy and configs.
+ *
+ * Note: Any unrecognized configs will be ignored.
+ * @param config any configs that should be applied to the changelog
+ * @return the factory to create a persistent key-value store
+ */
+ PersistentKeyValueFactory<K, V> enableLogging(final Map<String, String> config);
+
+ /**
+ * Indicates that a changelog should not be created for the key-value store
+ * @return the factory to create a persistent key-value store
+ */
+ PersistentKeyValueFactory<K, V> disableLogging();
+
+ /**
* Return the instance of StateStoreSupplier of new key-value store.
* @return the key-value store; never null
*/
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractStoreSupplier.java
new file mode 100644
index 0000000..64d6e07
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/AbstractStoreSupplier.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.processor.StateStoreSupplier;
+
+import java.util.Map;
+
+public abstract class AbstractStoreSupplier<K, V> implements StateStoreSupplier {
+ protected final String name;
+ protected final Serde<K> keySerde;
+ protected final Serde<V> valueSerde;
+ protected final Time time;
+ protected final boolean logged;
+ protected final Map<String, String> logConfig;
+
+ public AbstractStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, Time time, boolean logged, Map<String, String> logConfig) {
+ this.time = time;
+ this.name = name;
+ this.valueSerde = valueSerde;
+ this.keySerde = keySerde;
+ this.logged = logged;
+ this.logConfig = logConfig;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public Map<String, String> logConfig() {
+ return logConfig;
+ }
+
+ public boolean loggingEnabled() {
+ return logged;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
index 3953fd0..c05ebb2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
@@ -23,7 +23,6 @@ import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.StateRestoreCallback;
import org.apache.kafka.streams.processor.StateStore;
-import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.StateSerdes;
@@ -46,32 +45,21 @@ import java.util.TreeMap;
*
* @see org.apache.kafka.streams.state.Stores#create(String)
*/
-public class InMemoryKeyValueStoreSupplier<K, V> implements StateStoreSupplier {
+public class InMemoryKeyValueStoreSupplier<K, V> extends AbstractStoreSupplier<K, V> {
- private final String name;
- private final Time time;
- private final Serde<K> keySerde;
- private final Serde<V> valueSerde;
- public InMemoryKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde) {
- this(name, keySerde, valueSerde, null);
+ public InMemoryKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, boolean logged, Map<String, String> logConfig) {
+ this(name, keySerde, valueSerde, null, logged, logConfig);
}
- public InMemoryKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, Time time) {
- this.name = name;
- this.time = time;
- this.keySerde = keySerde;
- this.valueSerde = valueSerde;
- }
-
- public String name() {
- return name;
+ public InMemoryKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, Time time, boolean logged, Map<String, String> logConfig) {
+ super(name, keySerde, valueSerde, time, logged, logConfig);
}
public StateStore get() {
MemoryStore<K, V> store = new MemoryStore<>(name, keySerde, valueSerde);
- return new MeteredKeyValueStore<>(store.enableLogging(), "in-memory-state", time);
+ return new MeteredKeyValueStore<>(logged ? store.enableLogging() : store, "in-memory-state", time);
}
private static class MemoryStore<K, V> implements KeyValueStore<K, V> {
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreSupplier.java
index 20a7333..45bcca3 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryLRUCacheStoreSupplier.java
@@ -19,7 +19,8 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.streams.processor.StateStore;
-import org.apache.kafka.streams.processor.StateStoreSupplier;
+
+import java.util.Map;
/**
* An in-memory key-value store that is limited in size and retains a maximum number of most recently used entries.
@@ -28,34 +29,22 @@ import org.apache.kafka.streams.processor.StateStoreSupplier;
* @param <V> The value type
*
*/
-public class InMemoryLRUCacheStoreSupplier<K, V> implements StateStoreSupplier {
+public class InMemoryLRUCacheStoreSupplier<K, V> extends AbstractStoreSupplier<K, V> {
- private final String name;
private final int capacity;
- private final Serde<K> keySerde;
- private final Serde<V> valueSerde;
- private final Time time;
- public InMemoryLRUCacheStoreSupplier(String name, int capacity, Serde<K> keySerde, Serde<V> valueSerde) {
- this(name, capacity, keySerde, valueSerde, null);
+ public InMemoryLRUCacheStoreSupplier(String name, int capacity, Serde<K> keySerde, Serde<V> valueSerde, boolean logged, Map<String, String> logConfig) {
+ this(name, capacity, keySerde, valueSerde, null, logged, logConfig);
}
- public InMemoryLRUCacheStoreSupplier(String name, int capacity, Serde<K> keySerde, Serde<V> valueSerde, Time time) {
- this.name = name;
+ public InMemoryLRUCacheStoreSupplier(String name, int capacity, Serde<K> keySerde, Serde<V> valueSerde, Time time, boolean logged, Map<String, String> logConfig) {
+ super(name, keySerde, valueSerde, time, logged, logConfig);
this.capacity = capacity;
- this.keySerde = keySerde;
- this.valueSerde = valueSerde;
- this.time = time;
- }
-
- public String name() {
- return name;
}
public StateStore get() {
MemoryNavigableLRUCache<K, V> cache = new MemoryNavigableLRUCache<>(name, capacity, keySerde, valueSerde);
- InMemoryKeyValueLoggedStore<K, V> loggedCache = (InMemoryKeyValueLoggedStore<K, V>) cache.enableLogging();
-
- return new MeteredKeyValueStore<>(loggedCache, "in-memory-lru-state", time);
+ return new MeteredKeyValueStore<>(logged ? cache.enableLogging() : cache, "in-memory-lru-state", time);
}
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplier.java
index 16111ad..c10b7e1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBKeyValueStoreSupplier.java
@@ -20,7 +20,8 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.streams.processor.StateStore;
-import org.apache.kafka.streams.processor.StateStoreSupplier;
+
+import java.util.Map;
/**
* A {@link org.apache.kafka.streams.state.KeyValueStore} that stores all entries in a local RocksDB database.
@@ -30,31 +31,19 @@ import org.apache.kafka.streams.processor.StateStoreSupplier;
*
* @see org.apache.kafka.streams.state.Stores#create(String)
*/
-public class RocksDBKeyValueStoreSupplier<K, V> implements StateStoreSupplier {
-
- private final String name;
- private final Serde<K> keySerde;
- private final Serde<V> valueSerde;
- private final Time time;
-
- public RocksDBKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde) {
- this(name, keySerde, valueSerde, null);
- }
+public class RocksDBKeyValueStoreSupplier<K, V> extends AbstractStoreSupplier<K, V> {
- public RocksDBKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, Time time) {
- this.name = name;
- this.keySerde = keySerde;
- this.valueSerde = valueSerde;
- this.time = time;
+ public RocksDBKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, boolean logged, Map<String, String> logConfig) {
+ this(name, keySerde, valueSerde, null, logged, logConfig);
}
- public String name() {
- return name;
+ public RocksDBKeyValueStoreSupplier(String name, Serde<K> keySerde, Serde<V> valueSerde, Time time, boolean logged, Map<String, String> logConfig) {
+ super(name, keySerde, valueSerde, time, logged, logConfig);
}
public StateStore get() {
RocksDBStore<K, V> store = new RocksDBStore<>(name, keySerde, valueSerde);
-
- return new MeteredKeyValueStore<>(store.enableLogging(), "rocksdb-state", time);
+ return new MeteredKeyValueStore<>(logged ? store.enableLogging() : store, "rocksdb-state", time);
}
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
index 3a1bd59..107a5e4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
@@ -20,7 +20,8 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.streams.processor.StateStore;
-import org.apache.kafka.streams.processor.StateStoreSupplier;
+
+import java.util.Map;
/**
* A {@link org.apache.kafka.streams.state.KeyValueStore} that stores all entries in a local RocksDB database.
@@ -30,38 +31,30 @@ import org.apache.kafka.streams.processor.StateStoreSupplier;
*
* @see org.apache.kafka.streams.state.Stores#create(String)
*/
-public class RocksDBWindowStoreSupplier<K, V> implements StateStoreSupplier {
+public class RocksDBWindowStoreSupplier<K, V> extends AbstractStoreSupplier<K, V> {
- private final String name;
private final long retentionPeriod;
private final boolean retainDuplicates;
private final int numSegments;
- private final Serde<K> keySerde;
- private final Serde<V> valueSerde;
- private final Time time;
- public RocksDBWindowStoreSupplier(String name, long retentionPeriod, int numSegments, boolean retainDuplicates, Serde<K> keySerde, Serde<V> valueSerde) {
- this(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde, null);
+ public RocksDBWindowStoreSupplier(String name, long retentionPeriod, int numSegments, boolean retainDuplicates, Serde<K> keySerde, Serde<V> valueSerde, boolean logged, Map<String, String> logConfig) {
+ this(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde, null, logged, logConfig);
}
- public RocksDBWindowStoreSupplier(String name, long retentionPeriod, int numSegments, boolean retainDuplicates, Serde<K> keySerde, Serde<V> valueSerde, Time time) {
- this.name = name;
+ public RocksDBWindowStoreSupplier(String name, long retentionPeriod, int numSegments, boolean retainDuplicates, Serde<K> keySerde, Serde<V> valueSerde, Time time, boolean logged, Map<String, String> logConfig) {
+ super(name, keySerde, valueSerde, time, logged, logConfig);
this.retentionPeriod = retentionPeriod;
this.retainDuplicates = retainDuplicates;
this.numSegments = numSegments;
- this.keySerde = keySerde;
- this.valueSerde = valueSerde;
- this.time = time;
- }
-
- public String name() {
- return name;
}
public StateStore get() {
RocksDBWindowStore<K, V> store = new RocksDBWindowStore<>(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde);
- return new MeteredWindowStore<>(store.enableLogging(), "rocksdb-window", time);
+ return new MeteredWindowStore<>(logged ? store.enableLogging() : store, "rocksdb-window", time);
}
+ public long retentionPeriod() {
+ return retentionPeriod;
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/69ebf6f7/streams/src/main/java/org/apache/kafka/streams/state/internals/StoreChangeLogger.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/StoreChangeLogger.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/StoreChangeLogger.java
index 3f848fe..41f9ae2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/StoreChangeLogger.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/StoreChangeLogger.java
@@ -109,7 +109,7 @@ public class StoreChangeLogger<K, V> {
}
for (K k : this.dirty) {
V v = getter.get(k);
- collector.send(new ProducerRecord<>(this.topic, this.partition, k, v), keySerializer, valueSerializer);
+ collector.send(new ProducerRecord<>(this.topic, this.partition, context.timestamp(), k, v), keySerializer, valueSerializer);
}
this.removed.clear();
this.dirty.clear();