You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by gu...@apache.org on 2017/01/06 18:12:34 UTC
[1/5] kafka git commit: KAFKA-3452: Support session windows
Repository: kafka
Updated Branches:
refs/heads/trunk 00964ec74 -> e0de3a421
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
index b15ebab..fc30740 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreTest.java
@@ -60,13 +60,14 @@ public class RocksDBWindowStoreTest {
private final Serde<byte[]> byteArraySerde = Serdes.ByteArray();
private final String windowName = "window";
private final int numSegments = 3;
- private final long segmentSize = RocksDBWindowStore.MIN_SEGMENT_INTERVAL;
+ private final long segmentSize = Segments.MIN_SEGMENT_INTERVAL;
private final long retentionPeriod = segmentSize * (numSegments - 1);
private final long windowSize = 3;
private final Serde<Integer> intSerde = Serdes.Integer();
private final Serde<String> stringSerde = Serdes.String();
private final StateSerdes<Integer, String> serdes = new StateSerdes<>("", intSerde, stringSerde);
private static final long DEFAULT_CACHE_SIZE_BYTES = 1 * 1024 * 1024L;
+ private final Segments segments = new Segments(windowName, retentionPeriod, numSegments);
@SuppressWarnings("unchecked")
protected <K, V> WindowStore<K, V> createWindowStore(ProcessorContext context, final boolean enableCaching, final boolean retainDuplicates) {
@@ -449,7 +450,7 @@ public class RocksDBWindowStoreTest {
recordCollector, new ThreadCache(DEFAULT_CACHE_SIZE_BYTES));
WindowStore<Integer, String> store = createWindowStore(context, true, false);
- assertTrue(store instanceof CachingWindowStore);
+ assertTrue(store instanceof CachedStateStore);
} finally {
Utils.delete(baseDir);
}
@@ -480,30 +481,36 @@ public class RocksDBWindowStoreTest {
RocksDBWindowStore<Integer, String> inner =
(RocksDBWindowStore<Integer, String>) ((MeteredWindowStore<Integer, String>) store).inner();
try {
+ // to validate segments
+ final Segments segments = new Segments(windowName, retentionPeriod, numSegments);
long startTime = segmentSize * 2;
long incr = segmentSize / 2;
context.setRecordContext(createRecordContext(startTime));
store.put(0, "zero");
- assertEquals(Utils.mkSet(2L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(2)), segmentDirs(baseDir));
context.setRecordContext(createRecordContext(startTime + incr));
store.put(1, "one");
- assertEquals(Utils.mkSet(2L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(2)), segmentDirs(baseDir));
context.setRecordContext(createRecordContext(startTime + incr * 2));
store.put(2, "two");
- assertEquals(Utils.mkSet(2L, 3L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(2),
+ segments.segmentName(3)), segmentDirs(baseDir));
- // (3, "three") is not put
- assertEquals(Utils.mkSet(2L, 3L), inner.segmentIds());
context.setRecordContext(createRecordContext(startTime + incr * 4));
store.put(4, "four");
- assertEquals(Utils.mkSet(2L, 3L, 4L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(2),
+ segments.segmentName(3),
+ segments.segmentName(4)), segmentDirs(baseDir));
+
context.setRecordContext(createRecordContext(startTime + incr * 5));
store.put(5, "five");
- assertEquals(Utils.mkSet(2L, 3L, 4L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(2),
+ segments.segmentName(3),
+ segments.segmentName(4)), segmentDirs(baseDir));
assertEquals(Utils.mkList("zero"), toList(store.fetch(0, startTime - windowSize, startTime + windowSize)));
assertEquals(Utils.mkList("one"), toList(store.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
@@ -514,7 +521,10 @@ public class RocksDBWindowStoreTest {
context.setRecordContext(createRecordContext(startTime + incr * 6));
store.put(6, "six");
- assertEquals(Utils.mkSet(3L, 4L, 5L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(3),
+ segments.segmentName(4),
+ segments.segmentName(5)), segmentDirs(baseDir));
+
assertEquals(Utils.mkList(), toList(store.fetch(0, startTime - windowSize, startTime + windowSize)));
assertEquals(Utils.mkList(), toList(store.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
@@ -527,7 +537,9 @@ public class RocksDBWindowStoreTest {
context.setRecordContext(createRecordContext(startTime + incr * 7));
store.put(7, "seven");
- assertEquals(Utils.mkSet(3L, 4L, 5L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(3),
+ segments.segmentName(4),
+ segments.segmentName(5)), segmentDirs(baseDir));
assertEquals(Utils.mkList(), toList(store.fetch(0, startTime - windowSize, startTime + windowSize)));
assertEquals(Utils.mkList(), toList(store.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
@@ -540,7 +552,10 @@ public class RocksDBWindowStoreTest {
context.setRecordContext(createRecordContext(startTime + incr * 8));
store.put(8, "eight");
- assertEquals(Utils.mkSet(4L, 5L, 6L), inner.segmentIds());
+ assertEquals(Utils.mkSet(segments.segmentName(4),
+ segments.segmentName(5),
+ segments.segmentName(6)), segmentDirs(baseDir));
+
assertEquals(Utils.mkList(), toList(store.fetch(0, startTime - windowSize, startTime + windowSize)));
assertEquals(Utils.mkList(), toList(store.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
@@ -554,10 +569,10 @@ public class RocksDBWindowStoreTest {
// check segment directories
store.flush();
- assertEquals(
- Utils.mkSet(inner.segmentName(4L), inner.segmentName(5L), inner.segmentName(6L)),
- segmentDirs(baseDir)
- );
+ assertEquals(Utils.mkSet(segments.segmentName(4),
+ segments.segmentName(5),
+ segments.segmentName(6)), segmentDirs(baseDir));
+
} finally {
store.close();
}
@@ -567,6 +582,7 @@ public class RocksDBWindowStoreTest {
}
}
+
@Test
public void testRestore() throws IOException {
final List<KeyValue<byte[], byte[]>> changeLog = new ArrayList<>();
@@ -641,14 +657,10 @@ public class RocksDBWindowStoreTest {
recordCollector, new ThreadCache(DEFAULT_CACHE_SIZE_BYTES));
WindowStore<Integer, String> store = createWindowStore(context, false, true);
- RocksDBWindowStore<Integer, String> inner =
- (RocksDBWindowStore<Integer, String>) ((MeteredWindowStore<Integer, String>) store).inner();
try {
context.restore(windowName, changeLog);
- assertEquals(Utils.mkSet(4L, 5L, 6L), inner.segmentIds());
-
assertEquals(Utils.mkList(), toList(store.fetch(0, startTime - windowSize, startTime + windowSize)));
assertEquals(Utils.mkList(), toList(store.fetch(1, startTime + incr - windowSize, startTime + incr + windowSize)));
assertEquals(Utils.mkList(), toList(store.fetch(2, startTime + incr * 2 - windowSize, startTime + incr * 2 + windowSize)));
@@ -662,7 +674,7 @@ public class RocksDBWindowStoreTest {
// check segment directories
store.flush();
assertEquals(
- Utils.mkSet(inner.segmentName(4L), inner.segmentName(5L), inner.segmentName(6L)),
+ Utils.mkSet(segments.segmentName(4L), segments.segmentName(5L), segments.segmentName(6L)),
segmentDirs(baseDir)
);
} finally {
@@ -693,16 +705,13 @@ public class RocksDBWindowStoreTest {
recordCollector, new ThreadCache(DEFAULT_CACHE_SIZE_BYTES));
WindowStore<Integer, String> store = createWindowStore(context, false, true);
- RocksDBWindowStore<Integer, String> inner =
- (RocksDBWindowStore<Integer, String>) ((MeteredWindowStore<Integer, String>) store).inner();
-
try {
context.setTime(0L);
context.setRecordContext(createRecordContext(0));
store.put(0, "v");
assertEquals(
- Utils.mkSet(inner.segmentName(0L)),
+ Utils.mkSet(segments.segmentName(0L)),
segmentDirs(baseDir)
);
@@ -710,14 +719,14 @@ public class RocksDBWindowStoreTest {
store.put(0, "v");
store.put(0, "v");
assertEquals(
- Utils.mkSet(inner.segmentName(0L)),
+ Utils.mkSet(segments.segmentName(0L)),
segmentDirs(baseDir)
);
context.setRecordContext(createRecordContext(60000));
store.put(0, "v");
assertEquals(
- Utils.mkSet(inner.segmentName(0L), inner.segmentName(1L)),
+ Utils.mkSet(segments.segmentName(0L), segments.segmentName(1L)),
segmentDirs(baseDir)
);
@@ -733,7 +742,7 @@ public class RocksDBWindowStoreTest {
assertEquals(4, fetchedCount);
assertEquals(
- Utils.mkSet(inner.segmentName(0L), inner.segmentName(1L)),
+ Utils.mkSet(segments.segmentName(0L), segments.segmentName(1L)),
segmentDirs(baseDir)
);
@@ -749,7 +758,7 @@ public class RocksDBWindowStoreTest {
assertEquals(2, fetchedCount);
assertEquals(
- Utils.mkSet(inner.segmentName(1L), inner.segmentName(3L)),
+ Utils.mkSet(segments.segmentName(1L), segments.segmentName(3L)),
segmentDirs(baseDir)
);
@@ -765,7 +774,7 @@ public class RocksDBWindowStoreTest {
assertEquals(1, fetchedCount);
assertEquals(
- Utils.mkSet(inner.segmentName(3L), inner.segmentName(5L)),
+ Utils.mkSet(segments.segmentName(3L), segments.segmentName(5L)),
segmentDirs(baseDir)
);
@@ -798,27 +807,25 @@ public class RocksDBWindowStoreTest {
File storeDir = new File(baseDir, windowName);
WindowStore<Integer, String> store = createWindowStore(context, false, true);
- RocksDBWindowStore<Integer, String> inner =
- (RocksDBWindowStore<Integer, String>) ((MeteredWindowStore<Integer, String>) store).inner();
+
try {
- new File(storeDir, inner.segmentName(0L)).mkdir();
- new File(storeDir, inner.segmentName(1L)).mkdir();
- new File(storeDir, inner.segmentName(2L)).mkdir();
- new File(storeDir, inner.segmentName(3L)).mkdir();
- new File(storeDir, inner.segmentName(4L)).mkdir();
- new File(storeDir, inner.segmentName(5L)).mkdir();
- new File(storeDir, inner.segmentName(6L)).mkdir();
+ new File(storeDir, segments.segmentName(0L)).mkdir();
+ new File(storeDir, segments.segmentName(1L)).mkdir();
+ new File(storeDir, segments.segmentName(2L)).mkdir();
+ new File(storeDir, segments.segmentName(3L)).mkdir();
+ new File(storeDir, segments.segmentName(4L)).mkdir();
+ new File(storeDir, segments.segmentName(5L)).mkdir();
+ new File(storeDir, segments.segmentName(6L)).mkdir();
} finally {
store.close();
}
store = createWindowStore(context, false, true);
- inner = (RocksDBWindowStore<Integer, String>) ((MeteredWindowStore<Integer, String>) store).inner();
try {
assertEquals(
- Utils.mkSet(inner.segmentName(4L), inner.segmentName(5L), inner.segmentName(6L)),
+ Utils.mkSet(segments.segmentName(4L), segments.segmentName(5L), segments.segmentName(6L)),
segmentDirs(baseDir)
);
@@ -829,7 +836,7 @@ public class RocksDBWindowStoreTest {
}
assertEquals(
- Utils.mkSet(inner.segmentName(4L), inner.segmentName(5L), inner.segmentName(6L)),
+ Utils.mkSet(segments.segmentName(4L), segments.segmentName(5L), segments.segmentName(6L)),
segmentDirs(baseDir)
);
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
new file mode 100644
index 0000000..3869480
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentIteratorTest.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class SegmentIteratorTest {
+
+ private final Segment segmentOne = new Segment("one", "one", 0);
+ private final Segment segmentTwo = new Segment("two", "window", 1);
+ private final HasNextCondition hasNextCondition = new HasNextCondition() {
+ @Override
+ public boolean hasNext(final KeyValueIterator iterator) {
+ return iterator.hasNext();
+ }
+ };
+
+ @Before
+ public void before() {
+ final MockProcessorContext context = new MockProcessorContext(null,
+ TestUtils.tempDirectory(),
+ Serdes.String(),
+ Serdes.String(),
+ new NoOpRecordCollector(),
+ new ThreadCache(0));
+ segmentOne.openDB(context);
+ segmentTwo.openDB(context);
+ segmentOne.put(Bytes.wrap("a".getBytes()), "1".getBytes());
+ segmentOne.put(Bytes.wrap("b".getBytes()), "2".getBytes());
+ segmentTwo.put(Bytes.wrap("c".getBytes()), "3".getBytes());
+ segmentTwo.put(Bytes.wrap("d".getBytes()), "4".getBytes());
+
+ }
+
+ @After
+ public void closeSegments() {
+ segmentOne.close();
+ segmentTwo.close();
+ }
+
+ @Test
+ public void shouldIterateOverAllSegments() throws Exception {
+ final SegmentIterator iterator = new SegmentIterator(
+ Arrays.asList(segmentOne,
+ segmentTwo).iterator(),
+ hasNextCondition,
+ Bytes.wrap("a".getBytes()),
+ Bytes.wrap("z".getBytes()));
+
+ assertTrue(iterator.hasNext());
+ assertEquals("a", new String(iterator.peekNextKey().get()));
+ assertEquals(KeyValue.pair("a", "1"), toStringKeyValue(iterator.next()));
+
+ assertTrue(iterator.hasNext());
+ assertEquals("b", new String(iterator.peekNextKey().get()));
+ assertEquals(KeyValue.pair("b", "2"), toStringKeyValue(iterator.next()));
+
+ assertTrue(iterator.hasNext());
+ assertEquals("c", new String(iterator.peekNextKey().get()));
+ assertEquals(KeyValue.pair("c", "3"), toStringKeyValue(iterator.next()));
+
+ assertTrue(iterator.hasNext());
+ assertEquals("d", new String(iterator.peekNextKey().get()));
+ assertEquals(KeyValue.pair("d", "4"), toStringKeyValue(iterator.next()));
+
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test
+ public void shouldOnlyIterateOverSegmentsInRange() throws Exception {
+ final SegmentIterator iterator = new SegmentIterator(
+ Arrays.asList(segmentOne,
+ segmentTwo).iterator(),
+ hasNextCondition,
+ Bytes.wrap("a".getBytes()),
+ Bytes.wrap("b".getBytes()));
+
+ assertTrue(iterator.hasNext());
+ assertEquals("a", new String(iterator.peekNextKey().get()));
+ assertEquals(KeyValue.pair("a", "1"), toStringKeyValue(iterator.next()));
+
+ assertTrue(iterator.hasNext());
+ assertEquals("b", new String(iterator.peekNextKey().get()));
+ assertEquals(KeyValue.pair("b", "2"), toStringKeyValue(iterator.next()));
+
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void shouldThrowNoSuchElementOnPeekNextKeyIfNoNext() throws Exception {
+ final SegmentIterator iterator = new SegmentIterator(
+ Arrays.asList(segmentOne,
+ segmentTwo).iterator(),
+ hasNextCondition,
+ Bytes.wrap("f".getBytes()),
+ Bytes.wrap("h".getBytes()));
+
+ iterator.peekNextKey();
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void shouldThrowNoSuchElementOnNextIfNoNext() throws Exception {
+ final SegmentIterator iterator = new SegmentIterator(
+ Arrays.asList(segmentOne,
+ segmentTwo).iterator(),
+ hasNextCondition,
+ Bytes.wrap("f".getBytes()),
+ Bytes.wrap("h".getBytes()));
+
+ iterator.next();
+ }
+
+ private KeyValue<String, String> toStringKeyValue(final KeyValue<Bytes, byte[]> binaryKv) {
+ return KeyValue.pair(new String(binaryKv.key.get()), new String(binaryKv.value));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
new file mode 100644
index 0000000..9c2f688
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/SegmentsTest.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class SegmentsTest {
+
+ private static final int NUM_SEGMENTS = 5;
+ private MockProcessorContext context;
+ private Segments segments;
+
+ @Before
+ public void createContext() {
+ context = new MockProcessorContext(null,
+ TestUtils.tempDirectory(),
+ Serdes.String(),
+ Serdes.Long(),
+ new NoOpRecordCollector(),
+ new ThreadCache(0));
+ segments = new Segments("test", 4 * 60 * 1000, NUM_SEGMENTS);
+ }
+
+ @After
+ public void close() {
+ segments.close();
+ }
+
+ @Test
+ public void shouldGetSegmentIdsFromTimestamp() throws Exception {
+ assertEquals(0, segments.segmentId(0));
+ assertEquals(1, segments.segmentId(60000));
+ assertEquals(2, segments.segmentId(120000));
+ assertEquals(3, segments.segmentId(180000));
+ }
+
+ @Test
+ public void shouldBaseSegmentIntervalOnRetentionAndNumSegments() throws Exception {
+ final Segments segments = new Segments("test", 8 * 60 * 1000, 5);
+ assertEquals(0, segments.segmentId(0));
+ assertEquals(0, segments.segmentId(60000));
+ assertEquals(1, segments.segmentId(120000));
+ }
+
+ @Test
+ public void shouldGetSegmentNameFromId() throws Exception {
+ assertEquals("test-197001010000", segments.segmentName(0));
+ assertEquals("test-197001010001", segments.segmentName(1));
+ assertEquals("test-197001010002", segments.segmentName(2));
+ }
+
+ @Test
+ public void shouldCreateSegments() throws Exception {
+ final Segment segment1 = segments.getOrCreateSegment(0, context);
+ final Segment segment2 = segments.getOrCreateSegment(1, context);
+ final Segment segment3 = segments.getOrCreateSegment(2, context);
+ assertTrue(new File(context.stateDir(), "test/test-197001010000").isDirectory());
+ assertTrue(new File(context.stateDir(), "test/test-197001010001").isDirectory());
+ assertTrue(new File(context.stateDir(), "test/test-197001010002").isDirectory());
+ assertEquals(true, segment1.isOpen());
+ assertEquals(true, segment2.isOpen());
+ assertEquals(true, segment3.isOpen());
+ }
+
+ @Test
+ public void shouldNotCreateSegmentThatIsAlreadyExpired() throws Exception {
+ segments.getOrCreateSegment(7, context);
+ assertNull(segments.getOrCreateSegment(0, context));
+ assertFalse(new File(context.stateDir(), "test/test-197001010000").exists());
+ }
+
+ @Test
+ public void shouldCleanupSegmentsThatHaveExpired() throws Exception {
+ final Segment segment1 = segments.getOrCreateSegment(0, context);
+ final Segment segment2 = segments.getOrCreateSegment(0, context);
+ final Segment segment3 = segments.getOrCreateSegment(7, context);
+ assertFalse(segment1.isOpen());
+ assertFalse(segment2.isOpen());
+ assertTrue(segment3.isOpen());
+ assertFalse(new File(context.stateDir(), "test/test-197001010000").exists());
+ assertFalse(new File(context.stateDir(), "test/test-197001010001").exists());
+ assertTrue(new File(context.stateDir(), "test/test-197001010007").exists());
+ }
+
+ @Test
+ public void shouldGetSegmentForTimestamp() throws Exception {
+ final Segment segment = segments.getOrCreateSegment(0, context);
+ segments.getOrCreateSegment(1, context);
+ assertEquals(segment, segments.getSegmentForTimestamp(0L));
+ }
+
+ @Test
+ public void shouldCloseAllOpenSegments() throws Exception {
+ final Segment first = segments.getOrCreateSegment(0, context);
+ final Segment second = segments.getOrCreateSegment(1, context);
+ final Segment third = segments.getOrCreateSegment(2, context);
+ segments.close();
+
+ assertFalse(first.isOpen());
+ assertFalse(second.isOpen());
+ assertFalse(third.isOpen());
+ }
+
+ @Test
+ public void shouldOpenExistingSegments() throws Exception {
+ segments.getOrCreateSegment(0, context);
+ segments.getOrCreateSegment(1, context);
+ segments.getOrCreateSegment(2, context);
+ segments.getOrCreateSegment(3, context);
+ segments.getOrCreateSegment(4, context);
+ // close existing.
+ segments.close();
+
+ segments = new Segments("test", 4 * 60 * 1000, 5);
+ segments.openExisting(context);
+
+ assertTrue(segments.getSegmentForTimestamp(0).isOpen());
+ assertTrue(segments.getSegmentForTimestamp(1).isOpen());
+ assertTrue(segments.getSegmentForTimestamp(2).isOpen());
+ assertTrue(segments.getSegmentForTimestamp(3).isOpen());
+ assertTrue(segments.getSegmentForTimestamp(4).isOpen());
+ }
+
+ @Test
+ public void shouldGetSegmentsWithinTimeRange() throws Exception {
+ segments.getOrCreateSegment(0, context);
+ segments.getOrCreateSegment(1, context);
+ segments.getOrCreateSegment(2, context);
+ segments.getOrCreateSegment(3, context);
+ segments.getOrCreateSegment(4, context);
+
+ final List<Segment> segments = this.segments.segments(0, 2 * 60 * 1000);
+ assertEquals(3, segments.size());
+ assertEquals(0, segments.get(0).id);
+ assertEquals(1, segments.get(1).id);
+ assertEquals(2, segments.get(2).id);
+ }
+
+ @Test
+ public void shouldRollSegments() throws Exception {
+ segments.getOrCreateSegment(0, context);
+ verifyCorrectSegments(0, 1);
+ segments.getOrCreateSegment(1, context);
+ verifyCorrectSegments(0, 2);
+ segments.getOrCreateSegment(2, context);
+ verifyCorrectSegments(0, 3);
+ segments.getOrCreateSegment(3, context);
+ verifyCorrectSegments(0, 4);
+ segments.getOrCreateSegment(4, context);
+ verifyCorrectSegments(0, 5);
+ segments.getOrCreateSegment(5, context);
+ verifyCorrectSegments(1, 5);
+ segments.getOrCreateSegment(6, context);
+ verifyCorrectSegments(2, 5);
+ }
+
+ private void verifyCorrectSegments(final long first, final int numSegments) {
+ final List<Segment> result = this.segments.segments(0, Long.MAX_VALUE);
+ assertEquals(numSegments, result.size());
+ for (int i = 0; i < numSegments; i++) {
+ assertEquals(i + first, result.get(i).id);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
index 6c446ec..8aaf9e4 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ThreadCacheTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.kafka.streams.state.internals;
+import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KeyValue;
import org.junit.Test;
@@ -242,8 +243,8 @@ public class ThreadCacheTest {
final String namespace = "streams";
cache.put(namespace, theByte, dirtyEntry(theByte));
final ThreadCache.MemoryLRUCacheBytesIterator iterator = cache.range(namespace, theByte, new byte[]{1});
- assertArrayEquals(theByte, iterator.peekNextKey());
- assertArrayEquals(theByte, iterator.peekNextKey());
+ assertEquals(Bytes.wrap(theByte), iterator.peekNextKey());
+ assertEquals(Bytes.wrap(theByte), iterator.peekNextKey());
}
@Test
@@ -253,7 +254,7 @@ public class ThreadCacheTest {
final String namespace = "streams";
cache.put(namespace, theByte, dirtyEntry(theByte));
final ThreadCache.MemoryLRUCacheBytesIterator iterator = cache.range(namespace, theByte, new byte[]{1});
- assertArrayEquals(iterator.peekNextKey(), iterator.next().key);
+ assertEquals(iterator.peekNextKey(), iterator.next().key);
}
@Test(expected = NoSuchElementException.class)
@@ -281,10 +282,10 @@ public class ThreadCacheTest {
final ThreadCache.MemoryLRUCacheBytesIterator iterator = cache.range(namespace, new byte[]{1}, new byte[]{4});
int bytesIndex = 1;
while (iterator.hasNext()) {
- byte[] peekedKey = iterator.peekNextKey();
- final KeyValue<byte[], LRUCacheEntry> next = iterator.next();
- assertArrayEquals(bytes[bytesIndex], peekedKey);
- assertArrayEquals(bytes[bytesIndex], next.key);
+ Bytes peekedKey = iterator.peekNextKey();
+ final KeyValue<Bytes, LRUCacheEntry> next = iterator.next();
+ assertArrayEquals(bytes[bytesIndex], peekedKey.get());
+ assertArrayEquals(bytes[bytesIndex], next.key.get());
bytesIndex++;
}
assertEquals(5, bytesIndex);
@@ -311,7 +312,7 @@ public class ThreadCacheTest {
// should evict byte[] {0}
cache.put(namespace, new byte[]{6}, dirtyEntry(new byte[]{6}));
- assertArrayEquals(new byte[]{1}, range.peekNextKey());
+ assertEquals(Bytes.wrap(new byte[]{1}), range.peekNextKey());
}
@Test
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java b/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java
new file mode 100644
index 0000000..4642307
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/test/KeyValueIteratorStub.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.test;
+
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+import java.util.Iterator;
+
+public class KeyValueIteratorStub<K, V> implements KeyValueIterator<K, V> {
+
+ private final Iterator<KeyValue<K, V>> iterator;
+
+ public KeyValueIteratorStub(final Iterator<KeyValue<K, V>> iterator) {
+ this.iterator = iterator;
+ }
+
+ @Override
+ public void close() {
+ //no-op
+ }
+
+ @Override
+ public K peekNextKey() {
+ return null;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public KeyValue<K, V> next() {
+ return iterator.next();
+ }
+
+ @Override
+ public void remove() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java
index f4ab642..f058e30 100644
--- a/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java
+++ b/streams/src/test/java/org/apache/kafka/test/MockProcessorContext.java
@@ -21,6 +21,7 @@ import org.apache.kafka.common.metrics.Sensor;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsMetrics;
+import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
import org.apache.kafka.streams.processor.internals.ProcessorNode;
import org.apache.kafka.streams.processor.internals.RecordCollector;
@@ -95,6 +96,9 @@ public class MockProcessorContext implements InternalProcessorContext, RecordCol
}
public void setTime(long timestamp) {
+ if (recordContext != null) {
+ recordContext = new ProcessorRecordContext(timestamp, recordContext.offset(), recordContext.partition(), recordContext.topic());
+ }
this.timestamp = timestamp;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java
new file mode 100644
index 0000000..6169474
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/test/ReadOnlySessionStoreStub.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.test;
+
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.ReadOnlySessionStore;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ReadOnlySessionStoreStub<K, V> implements ReadOnlySessionStore<K, V>, StateStore {
+ private Map<K, List<KeyValue<Windowed<K>, V>>> sessions = new HashMap<>();
+ private boolean open = true;
+
+ public void put(final Windowed<K> sessionKey, final V value) {
+ if (!sessions.containsKey(sessionKey.key())) {
+ sessions.put(sessionKey.key(), new ArrayList<KeyValue<Windowed<K>, V>>());
+ }
+ sessions.get(sessionKey.key()).add(KeyValue.pair(sessionKey, value));
+ }
+
+ @Override
+ public KeyValueIterator<Windowed<K>, V> fetch(final K key) {
+ if (!open) {
+ throw new InvalidStateStoreException("not open");
+ }
+ if (!sessions.containsKey(key)) {
+ return new KeyValueIteratorStub<>(Collections.<KeyValue<Windowed<K>, V>>emptyIterator());
+ }
+ return new KeyValueIteratorStub<>(sessions.get(key).iterator());
+ }
+
+ @Override
+ public String name() {
+ return "";
+ }
+
+ @Override
+ public void init(final ProcessorContext context, final StateStore root) {
+
+ }
+
+ @Override
+ public void flush() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public boolean persistent() {
+ return false;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return open;
+ }
+
+
+ public void setOpen(final boolean open) {
+ this.open = open;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/test/SegmentedBytesStoreStub.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/test/SegmentedBytesStoreStub.java b/streams/src/test/java/org/apache/kafka/test/SegmentedBytesStoreStub.java
new file mode 100644
index 0000000..3092d47
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/test/SegmentedBytesStoreStub.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.test;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.internals.SegmentedBytesStore;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class SegmentedBytesStoreStub implements SegmentedBytesStore {
+ private Map<Bytes, byte[]> store = new HashMap<>();
+ public boolean fetchCalled;
+ public boolean allUpToCalled;
+ public boolean flushed;
+ public boolean closed;
+ public boolean initialized;
+ public boolean removeCalled;
+ public boolean putCalled;
+ public boolean getCalled;
+
+ @Override
+ public String name() {
+ return "";
+ }
+
+ @Override
+ public void init(final ProcessorContext context, final StateStore root) {
+ initialized = true;
+ }
+
+ @Override
+ public KeyValueIterator<Bytes, byte[]> fetch(final Bytes key, final long from, final long to) {
+ fetchCalled = true;
+ return new KeyValueIteratorStub<>(Collections.<KeyValue<Bytes, byte[]>>emptyIterator());
+ }
+
+ @Override
+ public void remove(final Bytes key) {
+ store.put(key, null);
+ removeCalled = true;
+ }
+
+ @Override
+ public void put(final Bytes key, final byte[] value) {
+ store.put(key, value);
+ putCalled = true;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return false;
+ }
+
+
+ @Override
+ public byte[] get(final Bytes key) {
+ getCalled = true;
+ return store.get(key);
+ }
+
+ @Override
+ public void flush() {
+ flushed = true;
+ }
+
+ @Override
+ public void close() {
+ closed = true;
+ }
+
+ @Override
+ public boolean persistent() {
+ return false;
+ }
+}
[3/5] kafka git commit: KAFKA-3452: Support session windows
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
index b563137..a2a420e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStore.java
@@ -21,159 +21,34 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.utils.Bytes;
-import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.errors.InvalidStateStoreException;
import org.apache.kafka.streams.processor.ProcessorContext;
-import org.apache.kafka.streams.processor.StateRestoreCallback;
import org.apache.kafka.streams.processor.StateStore;
import org.apache.kafka.streams.state.KeyValueIterator;
-import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.StateSerdes;
import org.apache.kafka.streams.state.WindowStore;
import org.apache.kafka.streams.state.WindowStoreIterator;
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.SimpleTimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-public class RocksDBWindowStore<K, V> implements WindowStore<K, V> {
-
- public static final long MIN_SEGMENT_INTERVAL = 60 * 1000; // one minute
-
- private volatile boolean open = false;
-
- // use the Bytes wrapper for underlying rocksDB keys since they are used for hashing data structures
- private static class Segment extends RocksDBStore<Bytes, byte[]> {
- public final long id;
-
- Segment(String segmentName, String windowName, long id) {
- super(segmentName, windowName, WindowStoreUtils.INNER_KEY_SERDE, WindowStoreUtils.INNER_VALUE_SERDE);
- this.id = id;
- }
-
- public void destroy() {
- Utils.delete(dbDir);
- }
-
- @Override
- public void openDB(final ProcessorContext context) {
- super.openDB(context);
- open = true;
- }
- }
-
- private static class RocksDBWindowStoreIterator<V> implements WindowStoreIterator<V> {
- private final StateSerdes<?, V> serdes;
- private final Iterator<Segment> segments;
- private final Bytes from;
- private final Bytes to;
- private KeyValueIterator<Bytes, byte[]> currentIterator;
- private KeyValueStore<Bytes, byte[]> currentSegment;
-
- RocksDBWindowStoreIterator(StateSerdes<?, V> serdes) {
- this(serdes, null, null, Collections.<Segment>emptyIterator());
- }
-
- RocksDBWindowStoreIterator(StateSerdes<?, V> serdes, Bytes from, Bytes to, Iterator<Segment> segments) {
- this.serdes = serdes;
- this.from = from;
- this.to = to;
- this.segments = segments;
- }
-
- @Override
- public boolean hasNext() {
- while ((currentIterator == null || !currentIterator.hasNext() || !currentSegment.isOpen())
- && segments.hasNext()) {
- close();
- currentSegment = segments.next();
- try {
- currentIterator = currentSegment.range(from, to);
- } catch (InvalidStateStoreException e) {
- // segment may have been closed so we ignore it.
- }
- }
- return currentIterator != null && currentIterator.hasNext();
- }
-
- /**
- * @throws NoSuchElementException if no next element exists
- */
- @Override
- public KeyValue<Long, V> next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- KeyValue<Bytes, byte[]> kv = currentIterator.next();
- return new KeyValue<>(WindowStoreUtils.timestampFromBinaryKey(kv.key.get()),
- serdes.valueFrom(kv.value));
- }
-
- @Override
- public void remove() {
-
- }
-
- @Override
- public void close() {
- if (currentIterator != null) {
- currentIterator.close();
- currentIterator = null;
- }
- }
- }
+class RocksDBWindowStore<K, V> implements WindowStore<K, V> {
private final String name;
- private final int numSegments;
- private final long segmentInterval;
+ private final SegmentedBytesStore bytesStore;
private final boolean retainDuplicates;
private final Serde<K> keySerde;
private final Serde<V> valueSerde;
- private final SimpleDateFormat formatter;
- private final ConcurrentHashMap<Long, Segment> segments = new ConcurrentHashMap<>();
-
private ProcessorContext context;
private int seqnum = 0;
- private long currentSegmentId = -1L;
-
private StateSerdes<K, V> serdes;
- private boolean loggingEnabled = false;
- private StoreChangeLogger<Bytes, byte[]> changeLogger = null;
- public RocksDBWindowStore(String name, long retentionPeriod, int numSegments, boolean retainDuplicates, Serde<K> keySerde, Serde<V> valueSerde) {
+ RocksDBWindowStore(String name, boolean retainDuplicates, Serde<K> keySerde, Serde<V> valueSerde, final SegmentedBytesStore bytesStore) {
this.name = name;
- this.numSegments = numSegments;
-
- // The segment interval must be greater than MIN_SEGMENT_INTERVAL
- this.segmentInterval = Math.max(retentionPeriod / (numSegments - 1), MIN_SEGMENT_INTERVAL);
-
this.keySerde = keySerde;
this.valueSerde = valueSerde;
-
this.retainDuplicates = retainDuplicates;
-
-
- // Create a date formatter. Formatted timestamps are used as segment name suffixes
- this.formatter = new SimpleDateFormat("yyyyMMddHHmm");
- this.formatter.setTimeZone(new SimpleTimeZone(0, "GMT"));
- }
-
- public RocksDBWindowStore<K, V> enableLogging() {
- loggingEnabled = true;
- return this;
+ this.bytesStore = bytesStore;
}
@@ -184,59 +59,14 @@ public class RocksDBWindowStore<K, V> implements WindowStore<K, V> {
@Override
@SuppressWarnings("unchecked")
- public void init(ProcessorContext context, StateStore root) {
+ public void init(final ProcessorContext context, final StateStore root) {
this.context = context;
-
// construct the serde
this.serdes = new StateSerdes<>(name,
- keySerde == null ? (Serde<K>) context.keySerde() : keySerde,
- valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde);
-
- openExistingSegments();
-
- this.changeLogger = this.loggingEnabled ? new StoreChangeLogger(name, context, WindowStoreUtils.INNER_SERDES) : null;
-
- // register and possibly restore the state from the logs
- context.register(root, loggingEnabled, new StateRestoreCallback() {
- @Override
- public void restore(byte[] key, byte[] value) {
- // if the value is null, it means that this record has already been
- // deleted while it was captured in the changelog, hence we do not need to put any more.
- if (value != null)
- putInternal(key, value);
- }
- });
-
- flush();
- open = true;
- }
-
- private void openExistingSegments() {
- try {
- File dir = new File(context.stateDir(), name);
+ keySerde == null ? (Serde<K>) context.keySerde() : keySerde,
+ valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde);
- if (dir.exists()) {
- String[] list = dir.list();
- if (list != null) {
- long[] segmentIds = new long[list.length];
- for (int i = 0; i < list.length; i++)
- segmentIds[i] = segmentIdFromSegmentName(list[i]);
-
- // open segments in the id order
- Arrays.sort(segmentIds);
- for (long segmentId : segmentIds) {
- if (segmentId >= 0) {
- currentSegmentId = segmentId;
- getOrCreateSegment(segmentId);
- }
- }
- }
- } else {
- dir.mkdir();
- }
- } catch (Exception ex) {
- // ignore
- }
+ bytesStore.init(context, root);
}
@Override
@@ -246,26 +76,17 @@ public class RocksDBWindowStore<K, V> implements WindowStore<K, V> {
@Override
public boolean isOpen() {
- return open;
+ return bytesStore.isOpen();
}
@Override
public void flush() {
- for (KeyValueStore<Bytes, byte[]> segment : segments.values()) {
- if (segment != null) {
- segment.flush();
- }
- }
+ bytesStore.flush();
}
@Override
public void close() {
- open = false;
- flush();
- for (KeyValueStore segment : segments.values()) {
- if (segment != null)
- segment.close();
- }
+ bytesStore.close();
}
@Override
@@ -275,166 +96,57 @@ public class RocksDBWindowStore<K, V> implements WindowStore<K, V> {
@Override
public void put(K key, V value, long timestamp) {
- final byte[] rawValue = serdes.rawValue(value);
- byte[] rawKey = putAndReturnInternalKey(key, rawValue, timestamp);
-
- if (rawKey != null && loggingEnabled) {
- changeLogger.logChange(Bytes.wrap(rawKey), rawValue);
- }
- }
-
- private byte[] putAndReturnInternalKey(K key, byte[] value, long timestamp) {
- long segmentId = segmentId(timestamp);
-
- if (segmentId > currentSegmentId) {
- // A new segment will be created. Clean up old segments first.
- currentSegmentId = segmentId;
- cleanup();
- }
-
- // If the record is within the retention period, put it in the store.
- KeyValueStore<Bytes, byte[]> segment = getOrCreateSegment(segmentId);
- if (segment != null) {
- if (retainDuplicates)
- seqnum = (seqnum + 1) & 0x7FFFFFFF;
- byte[] binaryKey = WindowStoreUtils.toBinaryKey(key, timestamp, seqnum, serdes);
- segment.put(Bytes.wrap(binaryKey), value);
- return binaryKey;
- } else {
- return null;
- }
- }
-
- private void putInternal(byte[] binaryKey, byte[] binaryValue) {
- final long timestamp = WindowStoreUtils.timestampFromBinaryKey(binaryKey);
- long segmentId = segmentId(timestamp);
-
- if (segmentId > currentSegmentId) {
- // A new segment will be created. Clean up old segments first.
- currentSegmentId = segmentId;
- cleanup();
- }
-
- // If the record is within the retention period, put it in the store.
- Segment segment = getOrCreateSegment(segmentId);
- if (segment != null) {
- segment.writeToStore(Bytes.wrap(binaryKey), binaryValue);
+ if (retainDuplicates) {
+ seqnum = (seqnum + 1) & 0x7FFFFFFF;
}
+ bytesStore.put(Bytes.wrap(WindowStoreUtils.toBinaryKey(key, timestamp, seqnum, serdes)), serdes.rawValue(value));
}
- private byte[] getInternal(byte[] binaryKey) {
- long segmentId = segmentId(WindowStoreUtils.timestampFromBinaryKey(binaryKey));
-
- KeyValueStore<Bytes, byte[]> segment = getSegment(segmentId);
- if (segment != null) {
- return segment.get(Bytes.wrap(binaryKey));
- } else {
- return null;
- }
- }
@SuppressWarnings("unchecked")
@Override
public WindowStoreIterator<V> fetch(K key, long timeFrom, long timeTo) {
- if (!isOpen()) {
- throw new InvalidStateStoreException("Store " + this.name + " is currently not isOpen");
- }
- long segFrom = segmentId(timeFrom);
- long segTo = segmentId(Math.max(0L, timeTo));
-
- byte[] binaryFrom = WindowStoreUtils.toBinaryKey(key, timeFrom, 0, serdes);
- byte[] binaryTo = WindowStoreUtils.toBinaryKey(key, timeTo, Integer.MAX_VALUE, serdes);
-
- final List<Segment> segments = new ArrayList<>();
- for (long segmentId = segFrom; segmentId <= segTo; segmentId++) {
- Segment segment = getSegment(segmentId);
- if (segment != null && segment.isOpen()) {
- try {
- segments.add(segment);
- } catch (InvalidStateStoreException ise) {
- // segment may have been closed by streams thread;
- }
- }
- }
-
- if (!segments.isEmpty()) {
- return new RocksDBWindowStoreIterator<>(serdes, Bytes.wrap(binaryFrom), Bytes.wrap(binaryTo), segments.iterator());
- } else {
- return new RocksDBWindowStoreIterator<>(serdes);
- }
- }
-
- private Segment getSegment(long segmentId) {
- final Segment segment = segments.get(segmentId % numSegments);
- if (!isSegment(segment, segmentId)) {
- return null;
- }
- return segment;
+ final KeyValueIterator<Bytes, byte[]> bytesIterator = bytesStore.fetch(Bytes.wrap(serdes.rawKey(key)), timeFrom, timeTo);
+ return new TheWindowStoreIterator<>(bytesIterator, serdes);
}
- private boolean isSegment(final Segment store, long segmentId) {
- return store != null && store.id == segmentId;
- }
+ private static class TheWindowStoreIterator<V> implements WindowStoreIterator<V> {
+ private final KeyValueIterator<Bytes, byte[]> actual;
+ private final StateSerdes<?, V> serdes;
- private Segment getOrCreateSegment(long segmentId) {
- if (segmentId <= currentSegmentId && segmentId > currentSegmentId - numSegments) {
- final long key = segmentId % numSegments;
- final Segment segment = segments.get(key);
- if (!isSegment(segment, segmentId)) {
- cleanup();
- }
- if (!segments.containsKey(key)) {
- Segment newSegment = new Segment(segmentName(segmentId), name, segmentId);
- newSegment.openDB(context);
- segments.put(key, newSegment);
- }
- return segments.get(key);
+ TheWindowStoreIterator(final KeyValueIterator<Bytes, byte[]> actual, final StateSerdes<?, V> serdes) {
+ this.actual = actual;
+ this.serdes = serdes;
+ }
- } else {
- return null;
+ @Override
+ public boolean hasNext() {
+ return actual.hasNext();
}
- }
- private void cleanup() {
- for (Map.Entry<Long, Segment> segmentEntry : segments.entrySet()) {
- final Segment segment = segmentEntry.getValue();
- if (segment != null && segment.id <= currentSegmentId - numSegments) {
- segments.remove(segmentEntry.getKey());
- segment.close();
- segment.destroy();
+ /**
+ * @throws NoSuchElementException if no next element exists
+ */
+ @Override
+ public KeyValue<Long, V> next() {
+ if (!actual.hasNext()) {
+ throw new NoSuchElementException();
}
+ final KeyValue<Bytes, byte[]> next = actual.next();
+ final long timestamp = WindowStoreUtils.timestampFromBinaryKey(next.key.get());
+ final V value = serdes.valueFrom(next.value);
+ return KeyValue.pair(timestamp, value);
}
- }
-
- private long segmentId(long timestamp) {
- return timestamp / segmentInterval;
- }
-
- // this method is defined public since it is used for unit tests
- public String segmentName(long segmentId) {
- return name + "-" + formatter.format(new Date(segmentId * segmentInterval));
- }
- public long segmentIdFromSegmentName(String segmentName) {
- try {
- Date date = formatter.parse(segmentName.substring(name.length() + 1));
- return date.getTime() / segmentInterval;
- } catch (Exception ex) {
- return -1L;
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
}
- }
- // this method is defined public since it is used for unit tests
- public Set<Long> segmentIds() {
- HashSet<Long> segmentIds = new HashSet<>();
-
- for (Segment segment : segments.values()) {
- if (segment != null) {
- segmentIds.add(segment.id);
- }
+ @Override
+ public void close() {
+ actual.close();
}
-
- return segmentIds;
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
index 49d8882..84f1734 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBWindowStoreSupplier.java
@@ -18,8 +18,6 @@
package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serde;
-import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.streams.state.WindowStore;
@@ -34,7 +32,7 @@ import java.util.Map;
* @see org.apache.kafka.streams.state.Stores#create(String)
*/
-public class RocksDBWindowStoreSupplier<K, V> extends AbstractStoreSupplier<K, V, WindowStore> {
+public class RocksDBWindowStoreSupplier<K, V> extends AbstractStoreSupplier<K, V, WindowStore> implements WindowStoreSupplier<WindowStore> {
private final long retentionPeriod;
private final boolean retainDuplicates;
@@ -60,15 +58,22 @@ public class RocksDBWindowStoreSupplier<K, V> extends AbstractStoreSupplier<K, V
}
public WindowStore get() {
+ final RocksDBSegmentedBytesStore bytesStore = new RocksDBSegmentedBytesStore(name, retentionPeriod, numSegments, new WindowStoreKeySchema());
if (!enableCaching) {
- final RocksDBWindowStore<K, V> rocksDbStore = new RocksDBWindowStore<>(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde);
- return new MeteredWindowStore<>(logged ? rocksDbStore.enableLogging() : rocksDbStore, "rocksdb-window", time);
+ final RocksDBWindowStore<K, V> segmentedStore = new RocksDBWindowStore<>(name, retainDuplicates, keySerde, valueSerde,
+ logged ? new ChangeLoggingSegmentedBytesStore(bytesStore)
+ : bytesStore);
+ return new MeteredWindowStore<>(segmentedStore, "rocksdb-window", time);
}
- final RocksDBWindowStore<Bytes, byte[]> store = new RocksDBWindowStore<>(name, retentionPeriod, numSegments, false, Serdes.Bytes(), Serdes.ByteArray());
- return new CachingWindowStore<>(new MeteredWindowStore<>(logged ? store.enableLogging() : store, "rocksdb-window", time), keySerde, valueSerde, windowSize);
+ return new CachingWindowStore<>(new MeteredSegmentedBytesStore(logged ? new ChangeLoggingSegmentedBytesStore(bytesStore)
+ : bytesStore,
+ "rocksdb-window",
+ time),
+ keySerde, valueSerde, windowSize);
}
+ @Override
public long retentionPeriod() {
return retentionPeriod;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/Segment.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/Segment.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/Segment.java
new file mode 100644
index 0000000..5471262
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/Segment.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.processor.ProcessorContext;
+
+// use the Bytes wrapper for underlying rocksDB keys since they are used for hashing data structures
+class Segment extends RocksDBStore<Bytes, byte[]> {
+ public final long id;
+
+ Segment(String segmentName, String windowName, long id) {
+ super(segmentName, windowName, WindowStoreUtils.INNER_KEY_SERDE, WindowStoreUtils.INNER_VALUE_SERDE);
+ this.id = id;
+ }
+
+ public void destroy() {
+ Utils.delete(dbDir);
+ }
+
+ @Override
+ public void openDB(final ProcessorContext context) {
+ super.openDB(context);
+ open = true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java
new file mode 100644
index 0000000..ed2da9c
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentIterator.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.KeyValueStore;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Iterate over multiple Segments
+ * @param <K>
+ * @param <V>
+ */
+class SegmentIterator implements KeyValueIterator<Bytes, byte[]> {
+
+ private final Iterator<Segment> segments;
+ private final HasNextCondition hasNextCondition;
+ private final Bytes from;
+ private final Bytes to;
+ private KeyValueIterator<Bytes, byte[]> currentIterator;
+ private KeyValueStore<Bytes, byte[]> currentSegment;
+
+ SegmentIterator(final Iterator<Segment> segments,
+ final HasNextCondition hasNextCondition,
+ final Bytes from,
+ final Bytes to) {
+ this.segments = segments;
+ this.hasNextCondition = hasNextCondition;
+ this.from = from;
+ this.to = to;
+ }
+
+ public void close() {
+ if (currentIterator != null) {
+ currentIterator.close();
+ currentIterator = null;
+ }
+ }
+
+ @Override
+ public Bytes peekNextKey() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return currentIterator.peekNextKey();
+ }
+
+ @SuppressWarnings("unchecked")
+ public boolean hasNext() {
+ boolean hasNext = false;
+ while ((currentIterator == null || !(hasNext = hasNextCondition.hasNext(currentIterator)) || !currentSegment.isOpen())
+ && segments.hasNext()) {
+ close();
+ currentSegment = segments.next();
+ try {
+ currentIterator = currentSegment.range(from, to);
+ } catch (InvalidStateStoreException e) {
+ // segment may have been closed so we ignore it.
+ }
+ }
+ return currentIterator != null && hasNext;
+ }
+
+ public KeyValue<Bytes, byte[]> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return currentIterator.next();
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("remove not supported");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java
new file mode 100644
index 0000000..ab1099e
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SegmentedBytesStore.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+import java.util.List;
+
+/**
+ * The interface representing a StateStore that has 1 or more segments that are based
+ * on time.
+ * @see RocksDBSegmentedBytesStore
+ */
+public interface SegmentedBytesStore extends StateStore {
+
+ /**
+ * Fetch all records from the segmented store with the provided key and time range
+ * from all existing segments
+ * @param key the key to match
+ * @param from earliest time to match
+ * @param to latest time to match
+ * @return an iterator over key-value pairs
+ */
+ KeyValueIterator<Bytes, byte[]> fetch(Bytes key, long from, long to);
+
+ /**
+ * Remove the record with the provided key. The key
+ * should be a composite of the record key, and the timestamp information etc
+ * as described by the {@link KeySchema}
+ * @param key the segmented key to remove
+ */
+ void remove(Bytes key);
+
+ /**
+ * Write a new value to the store with the provided key. The key
+ * should be a composite of the record key, and the timestamp information etc
+ * as described by the {@link KeySchema}
+ * @param key
+ * @param value
+ */
+ void put(Bytes key, byte[] value);
+
+ /**
+ * Get the record from the store with the given key. The key
+ * should be a composite of the record key, and the timestamp information etc
+ * as described by the {@link KeySchema}
+ * @param key
+ * @return
+ */
+ byte[] get(Bytes key);
+
+
+ interface KeySchema {
+ /**
+ * Given a record-key and a time, construct a Segmented key that represents
+ * the upper range of keys to search when performing range queries.
+ * @see SessionKeySchema#upperRange
+ * @see WindowStoreKeySchema#upperRange
+ * @param key
+ * @param to
+ * @return The key that represents the upper range to search for in the store
+ */
+ Bytes upperRange(final Bytes key, final long to);
+
+ /**
+ * Given a record-key and a time, construct a Segmented key that represents
+ * the lower range of keys to search when performing range queries.
+ * @see SessionKeySchema#lowerRange
+ * @see WindowStoreKeySchema#lowerRange
+ * @param key
+ * @param from
+ * @return The key that represents the lower range to search for in the store
+ */
+ Bytes lowerRange(final Bytes key, final long from);
+
+ /**
+ * Extract the timestamp of the segment from the key. The key is a composite of
+ * the record-key, any timestamps, plus any additional information.
+ * @see SessionKeySchema#lowerRange
+ * @see WindowStoreKeySchema#lowerRange
+ * @param key
+ * @return
+ */
+ long segmentTimestamp(final Bytes key);
+
+ /**
+ * Create an implementation of {@link HasNextCondition} that knows when
+ * to stop iterating over the Segments. Used during {@link SegmentedBytesStore#fetch(Bytes, long, long)} operations
+ * @param binaryKey the record-key
+ * @param from starting time range
+ * @param to ending time range
+ * @return
+ */
+ HasNextCondition hasNextCondition(final Bytes binaryKey, long from, long to);
+
+ /**
+ * Used during {@link SegmentedBytesStore#fetch(Bytes, long, long)} operations to determine
+ * which segments should be scanned.
+ * @param segments
+ * @param from
+ * @param to
+ * @return List of segments to search
+ */
+ List<Segment> segmentsToSearch(Segments segments, long from, long to);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/Segments.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/Segments.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/Segments.java
new file mode 100644
index 0000000..13cdf6c
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/Segments.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.errors.ProcessorStateException;
+import org.apache.kafka.streams.processor.ProcessorContext;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.SimpleTimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Manages the {@link Segment}s that are used by the {@link RocksDBSegmentedBytesStore}
+ */
+class Segments {
+ static final long MIN_SEGMENT_INTERVAL = 60 * 1000L;
+ private final ConcurrentHashMap<Long, Segment> segments = new ConcurrentHashMap<>();
+ private final String name;
+ private final int numSegments;
+ private final long segmentInterval;
+ private final SimpleDateFormat formatter;
+ private long currentSegmentId = -1L;
+
+ Segments(final String name, final long retentionPeriod, final int numSegments) {
+ this.name = name;
+ this.numSegments = numSegments;
+ this.segmentInterval = Math.max(retentionPeriod / (numSegments - 1), MIN_SEGMENT_INTERVAL);
+ // Create a date formatter. Formatted timestamps are used as segment name suffixes
+ this.formatter = new SimpleDateFormat("yyyyMMddHHmm");
+ this.formatter.setTimeZone(new SimpleTimeZone(0, "UTC"));
+ }
+
+ long segmentId(long timestamp) {
+ return timestamp / segmentInterval;
+ }
+
+ String segmentName(long segmentId) {
+ return name + "-" + formatter.format(new Date(segmentId * segmentInterval));
+ }
+
+ Segment getSegmentForTimestamp(final long timestamp) {
+ return getSegment(segmentId(timestamp));
+ }
+
+ Segment getOrCreateSegment(final long segmentId, final ProcessorContext context) {
+ if (segmentId > currentSegmentId || segmentId > currentSegmentId - numSegments) {
+ final long key = segmentId % numSegments;
+ final Segment segment = segments.get(key);
+ if (!isSegment(segment, segmentId)) {
+ cleanup(segmentId);
+ }
+ if (!segments.containsKey(key)) {
+ Segment newSegment = new Segment(segmentName(segmentId), name, segmentId);
+ newSegment.openDB(context);
+ segments.put(key, newSegment);
+ currentSegmentId = segmentId > currentSegmentId ? segmentId : currentSegmentId;
+ }
+ return segments.get(key);
+ } else {
+ return null;
+ }
+ }
+
+
+ void openExisting(final ProcessorContext context) {
+ try {
+ File dir = new File(context.stateDir(), name);
+ if (dir.exists()) {
+ String[] list = dir.list();
+ if (list != null) {
+ long[] segmentIds = new long[list.length];
+ for (int i = 0; i < list.length; i++)
+ segmentIds[i] = segmentIdFromSegmentName(list[i]);
+
+ // open segments in the id order
+ Arrays.sort(segmentIds);
+ for (long segmentId : segmentIds) {
+ if (segmentId >= 0) {
+ getOrCreateSegment(segmentId, context);
+ }
+ }
+ }
+ } else {
+ if (!dir.mkdir()) {
+ throw new ProcessorStateException(String.format("dir %s doesn't exist and cannot be created for store %s", dir, name));
+ }
+ }
+ } catch (Exception ex) {
+ // ignore
+ }
+ }
+
+ List<Segment> segments(final long timeFrom, final long timeTo) {
+ final long segFrom = segmentId(Math.max(0L, timeFrom));
+ final long segTo = segmentId(Math.min(currentSegmentId * segmentInterval, Math.max(0, timeTo)));
+
+ final List<Segment> segments = new ArrayList<>();
+ for (long segmentId = segFrom; segmentId <= segTo; segmentId++) {
+ Segment segment = getSegment(segmentId);
+ if (segment != null && segment.isOpen()) {
+ try {
+ segments.add(segment);
+ } catch (InvalidStateStoreException ise) {
+ // segment may have been closed by streams thread;
+ }
+ }
+ }
+ return segments;
+ }
+
+ void flush() {
+ for (Segment segment : segments.values()) {
+ segment.flush();
+ }
+ }
+
+ public void close() {
+ for (Segment segment : segments.values()) {
+ segment.close();
+ }
+ }
+
+ private Segment getSegment(long segmentId) {
+ final Segment segment = segments.get(segmentId % numSegments);
+ if (!isSegment(segment, segmentId)) {
+ return null;
+ }
+ return segment;
+ }
+
+ private boolean isSegment(final Segment store, long segmentId) {
+ return store != null && store.id == segmentId;
+ }
+
+ private void cleanup(final long segmentId) {
+ final long oldestSegmentId = currentSegmentId < segmentId
+ ? segmentId - numSegments
+ : currentSegmentId - numSegments;
+
+ for (Map.Entry<Long, Segment> segmentEntry : segments.entrySet()) {
+ final Segment segment = segmentEntry.getValue();
+ if (segment != null && segment.id <= oldestSegmentId) {
+ segments.remove(segmentEntry.getKey());
+ segment.close();
+ segment.destroy();
+ }
+ }
+ }
+
+ private long segmentIdFromSegmentName(String segmentName) {
+ try {
+ Date date = formatter.parse(segmentName.substring(name.length() + 1));
+ return date.getTime() / segmentInterval;
+ } catch (Exception ex) {
+ return -1L;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java
new file mode 100644
index 0000000..b15eec9
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/SessionKeySchema.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.SessionKeySerde;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+import java.util.List;
+
+
+class SessionKeySchema implements SegmentedBytesStore.KeySchema {
+
+ @Override
+ public Bytes upperRange(final Bytes key, final long to) {
+ final Windowed<Bytes> sessionKey = new Windowed<>(key, new TimeWindow(to, Long.MAX_VALUE));
+ return SessionKeySerde.toBinary(sessionKey, Serdes.Bytes().serializer());
+ }
+
+ @Override
+ public Bytes lowerRange(final Bytes key, final long from) {
+ final Windowed<Bytes> sessionKey = new Windowed<>(key, new TimeWindow(0, Math.max(0, from)));
+ return SessionKeySerde.toBinary(sessionKey, Serdes.Bytes().serializer());
+ }
+
+ @Override
+ public long segmentTimestamp(final Bytes key) {
+ return SessionKeySerde.extractEnd(key.get());
+ }
+
+ @Override
+ public HasNextCondition hasNextCondition(final Bytes binaryKey, final long from, final long to) {
+ return new HasNextCondition() {
+ @Override
+ public boolean hasNext(final KeyValueIterator<Bytes, ?> iterator) {
+ if (iterator.hasNext()) {
+ final Bytes bytes = iterator.peekNextKey();
+ final Bytes keyBytes = Bytes.wrap(SessionKeySerde.extractKeyBytes(bytes.get()));
+ if (!keyBytes.equals(binaryKey)) {
+ return false;
+ }
+ final long start = SessionKeySerde.extractStart(bytes.get());
+ final long end = SessionKeySerde.extractEnd(bytes.get());
+ return end >= from && start <= to;
+ }
+ return false;
+ }
+ };
+ }
+
+ @Override
+ public List<Segment> segmentsToSearch(final Segments segments, final long from, final long to) {
+ return segments.segments(from, Long.MAX_VALUE);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
index c6c3030..a66c961 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java
@@ -234,24 +234,25 @@ public class ThreadCache {
}
- static class MemoryLRUCacheBytesIterator implements PeekingKeyValueIterator<byte[], LRUCacheEntry> {
+ static class MemoryLRUCacheBytesIterator implements PeekingKeyValueIterator<Bytes, LRUCacheEntry> {
private final Iterator<Bytes> keys;
private final NamedCache cache;
- private KeyValue<byte[], LRUCacheEntry> nextEntry;
+ private KeyValue<Bytes, LRUCacheEntry> nextEntry;
MemoryLRUCacheBytesIterator(final Iterator<Bytes> keys, final NamedCache cache) {
this.keys = keys;
this.cache = cache;
}
- public byte[] peekNextKey() {
+ public Bytes peekNextKey() {
if (!hasNext()) {
throw new NoSuchElementException();
}
return nextEntry.key;
}
- KeyValue<byte[], LRUCacheEntry> peekNext() {
+
+ public KeyValue<Bytes, LRUCacheEntry> peekNext() {
if (!hasNext()) {
throw new NoSuchElementException();
}
@@ -272,11 +273,11 @@ public class ThreadCache {
}
@Override
- public KeyValue<byte[], LRUCacheEntry> next() {
+ public KeyValue<Bytes, LRUCacheEntry> next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
- final KeyValue<byte[], LRUCacheEntry> result = nextEntry;
+ final KeyValue<Bytes, LRUCacheEntry> result = nextEntry;
nextEntry = null;
return result;
}
@@ -288,7 +289,7 @@ public class ThreadCache {
return;
}
- nextEntry = new KeyValue<>(cacheKey.get(), entry);
+ nextEntry = new KeyValue<>(cacheKey, entry);
}
@Override
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreKeySchema.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreKeySchema.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreKeySchema.java
new file mode 100644
index 0000000..093161e
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreKeySchema.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.StateSerdes;
+
+import java.util.List;
+
+class WindowStoreKeySchema implements RocksDBSegmentedBytesStore.KeySchema {
+ private static final HasNextCondition ITERATOR_HAS_NEXT = new HasNextCondition() {
+ @Override
+ public boolean hasNext(final KeyValueIterator<Bytes, ?> iterator) {
+ return iterator.hasNext();
+ }
+ };
+ private final StateSerdes<Bytes, byte[]> serdes = new StateSerdes<>("window-store-key-schema", Serdes.Bytes(), Serdes.ByteArray());
+
+ @Override
+ public Bytes upperRange(final Bytes key, final long to) {
+ return Bytes.wrap(WindowStoreUtils.toBinaryKey(key, to, Integer.MAX_VALUE, serdes));
+ }
+
+ @Override
+ public Bytes lowerRange(final Bytes key, final long from) {
+ return Bytes.wrap(WindowStoreUtils.toBinaryKey(key, Math.max(0, from), 0, serdes));
+ }
+
+ @Override
+ public long segmentTimestamp(final Bytes key) {
+ return WindowStoreUtils.timestampFromBinaryKey(key.get());
+ }
+
+ @Override
+ public HasNextCondition hasNextCondition(final Bytes binaryKey, final long from, final long to) {
+ return ITERATOR_HAS_NEXT;
+ }
+
+ @Override
+ public List<Segment> segmentsToSearch(final Segments segments, final long from, final long to) {
+ return segments.segments(from, to);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreSupplier.java
new file mode 100644
index 0000000..9c93dcc
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/WindowStoreSupplier.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.processor.StateStoreSupplier;
+
+public interface WindowStoreSupplier<T extends StateStore> extends StateStoreSupplier<T> {
+ long retentionPeriod();
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/integration/JoinIntegrationTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/JoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/JoinIntegrationTest.java
index 0f70588..b443abc 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/JoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/JoinIntegrationTest.java
@@ -165,7 +165,7 @@ public class JoinIntegrationTest {
private void checkResult(final String outputTopic, final List<String> expectedResult) throws Exception {
if (expectedResult != null) {
- final List<String> result = IntegrationTestUtils.waitUntilMinValuesRecordsReceived(RESULT_CONSUMER_CONFIG, outputTopic, expectedResult.size(), Long.MAX_VALUE);
+ final List<String> result = IntegrationTestUtils.waitUntilMinValuesRecordsReceived(RESULT_CONSUMER_CONFIG, outputTopic, expectedResult.size(), 30 * 1000L);
assertThat(result, is(expectedResult));
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
index bdbc1c8..cac8384 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java
@@ -29,14 +29,20 @@ import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
import org.apache.kafka.streams.kstream.Aggregator;
+import org.apache.kafka.streams.kstream.ForeachAction;
import org.apache.kafka.streams.kstream.Initializer;
import org.apache.kafka.streams.kstream.KGroupedStream;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KStreamBuilder;
import org.apache.kafka.streams.kstream.KeyValueMapper;
import org.apache.kafka.streams.kstream.Reducer;
+import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;
import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.QueryableStoreTypes;
+import org.apache.kafka.streams.state.ReadOnlySessionStore;
import org.apache.kafka.test.MockKeyValueMapper;
import org.apache.kafka.test.TestUtils;
import org.junit.After;
@@ -48,12 +54,18 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
public class KStreamAggregationIntegrationTest {
private static final int NUM_BROKERS = 1;
@@ -68,6 +80,7 @@ public class KStreamAggregationIntegrationTest {
private Properties streamsConfiguration;
private KafkaStreams kafkaStreams;
private String streamOneInput;
+ private String userSessionsStream = "user-sessions";
private String outputTopic;
private KGroupedStream<String, String> groupedStream;
private Reducer<String> reducer;
@@ -416,6 +429,194 @@ public class KStreamAggregationIntegrationTest {
}
+ @Test
+ public void shouldCountSessionWindows() throws Exception {
+ final long sessionGap = 5 * 60 * 1000L;
+ final long maintainMillis = sessionGap * 3;
+
+ final long t1 = mockTime.milliseconds() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
+ final List<KeyValue<String, String>> t1Messages = Arrays.asList(new KeyValue<>("bob", "start"),
+ new KeyValue<>("penny", "start"),
+ new KeyValue<>("jo", "pause"),
+ new KeyValue<>("emily", "pause"));
+
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ t1Messages,
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t1);
+
+ final long t2 = t1 + (sessionGap / 2);
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ Collections.singletonList(
+ new KeyValue<>("emily", "resume")
+ ),
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t2);
+ final long t3 = t1 + sessionGap + 1;
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ Arrays.asList(
+ new KeyValue<>("bob", "pause"),
+ new KeyValue<>("penny", "stop")
+ ),
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t3);
+
+ final long t4 = t3 + (sessionGap / 2);
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ Arrays.asList(
+ new KeyValue<>("bob", "resume"), // bobs session continues
+ new KeyValue<>("jo", "resume") // jo's starts new session
+ ),
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t4);
+
+ final Map<Windowed<String>, Long> results = new HashMap<>();
+ final CountDownLatch latch = new CountDownLatch(11);
+ builder.stream(Serdes.String(), Serdes.String(), userSessionsStream)
+ .groupByKey(Serdes.String(), Serdes.String())
+ .count(SessionWindows.with(sessionGap).until(maintainMillis), "UserSessionsStore")
+ .toStream()
+ .foreach(new ForeachAction<Windowed<String>, Long>() {
+ @Override
+ public void apply(final Windowed<String> key, final Long value) {
+ results.put(key, value);
+ latch.countDown();
+ }
+ });
+
+ startStreams();
+ latch.await(30, TimeUnit.SECONDS);
+ assertThat(results.get(new Windowed<>("bob", new TimeWindow(t1, t1))), equalTo(1L));
+ assertThat(results.get(new Windowed<>("penny", new TimeWindow(t1, t1))), equalTo(1L));
+ assertThat(results.get(new Windowed<>("jo", new TimeWindow(t1, t1))), equalTo(1L));
+ assertThat(results.get(new Windowed<>("jo", new TimeWindow(t4, t4))), equalTo(1L));
+ assertThat(results.get(new Windowed<>("emily", new TimeWindow(t1, t2))), equalTo(2L));
+ assertThat(results.get(new Windowed<>("bob", new TimeWindow(t3, t4))), equalTo(2L));
+ assertThat(results.get(new Windowed<>("penny", new TimeWindow(t3, t3))), equalTo(1L));
+ }
+
+ @Test
+ public void shouldReduceSessionWindows() throws Exception {
+ final long sessionGap = 1000L; // something to do with time
+ final long maintainMillis = sessionGap * 3;
+
+ final long t1 = mockTime.milliseconds();
+ final List<KeyValue<String, String>> t1Messages = Arrays.asList(new KeyValue<>("bob", "start"),
+ new KeyValue<>("penny", "start"),
+ new KeyValue<>("jo", "pause"),
+ new KeyValue<>("emily", "pause"));
+
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ t1Messages,
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t1);
+
+ final long t2 = t1 + (sessionGap / 2);
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ Collections.singletonList(
+ new KeyValue<>("emily", "resume")
+ ),
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t2);
+ final long t3 = t1 + sessionGap + 1;
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ Arrays.asList(
+ new KeyValue<>("bob", "pause"),
+ new KeyValue<>("penny", "stop")
+ ),
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t3);
+
+ final long t4 = t3 + (sessionGap / 2);
+ IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+ userSessionsStream,
+ Arrays.asList(
+ new KeyValue<>("bob", "resume"), // bobs session continues
+ new KeyValue<>("jo", "resume") // jo's starts new session
+ ),
+ TestUtils.producerConfig(
+ CLUSTER.bootstrapServers(),
+ StringSerializer.class,
+ StringSerializer.class,
+ new Properties()),
+ t4);
+
+ final Map<Windowed<String>, String> results = new HashMap<>();
+ final CountDownLatch latch = new CountDownLatch(11);
+ final String userSessionsStore = "UserSessionsStore";
+ builder.stream(Serdes.String(), Serdes.String(), userSessionsStream)
+ .groupByKey(Serdes.String(), Serdes.String())
+ .reduce(new Reducer<String>() {
+ @Override
+ public String apply(final String value1, final String value2) {
+ return value1 + ":" + value2;
+ }
+ }, SessionWindows.with(sessionGap).until(maintainMillis), userSessionsStore)
+ .foreach(new ForeachAction<Windowed<String>, String>() {
+ @Override
+ public void apply(final Windowed<String> key, final String value) {
+ results.put(key, value);
+ latch.countDown();
+ }
+ });
+
+ startStreams();
+ latch.await(30, TimeUnit.SECONDS);
+ final ReadOnlySessionStore<String, String> sessionStore
+ = kafkaStreams.store(userSessionsStore, QueryableStoreTypes.<String, String>sessionStore());
+
+ // verify correct data received
+ assertThat(results.get(new Windowed<>("bob", new TimeWindow(t1, t1))), equalTo("start"));
+ assertThat(results.get(new Windowed<>("penny", new TimeWindow(t1, t1))), equalTo("start"));
+ assertThat(results.get(new Windowed<>("jo", new TimeWindow(t1, t1))), equalTo("pause"));
+ assertThat(results.get(new Windowed<>("jo", new TimeWindow(t4, t4))), equalTo("resume"));
+ assertThat(results.get(new Windowed<>("emily", new TimeWindow(t1, t2))), equalTo("pause:resume"));
+ assertThat(results.get(new Windowed<>("bob", new TimeWindow(t3, t4))), equalTo("pause:resume"));
+ assertThat(results.get(new Windowed<>("penny", new TimeWindow(t3, t3))), equalTo("stop"));
+
+ // verify can query data via IQ
+ final KeyValueIterator<Windowed<String>, String> bob = sessionStore.fetch("bob");
+ assertThat(bob.next(), equalTo(KeyValue.pair(new Windowed<>("bob", new TimeWindow(t1, t1)), "start")));
+ assertThat(bob.next(), equalTo(KeyValue.pair(new Windowed<>("bob", new TimeWindow(t3, t4)), "pause:resume")));
+ assertFalse(bob.hasNext());
+
+ }
+
private void produceMessages(final long timestamp)
throws ExecutionException, InterruptedException {
@@ -439,7 +640,9 @@ public class KStreamAggregationIntegrationTest {
private void createTopics() {
streamOneInput = "stream-one-" + testNo;
outputTopic = "output-" + testNo;
+ userSessionsStream = userSessionsStream + "-" + testNo;
CLUSTER.createTopic(streamOneInput, 3, 1);
+ CLUSTER.createTopic(userSessionsStream);
CLUSTER.createTopic(outputTopic);
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
index 292f229..82e8c6c 100644
--- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java
@@ -18,25 +18,43 @@
package org.apache.kafka.streams.kstream.internals;
import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.kstream.Aggregator;
+import org.apache.kafka.streams.kstream.ForeachAction;
+import org.apache.kafka.streams.kstream.Initializer;
import org.apache.kafka.streams.kstream.KGroupedStream;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KStreamBuilder;
+import org.apache.kafka.streams.kstream.Reducer;
+import org.apache.kafka.streams.kstream.Merger;
+import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;
import org.apache.kafka.streams.processor.StateStoreSupplier;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.Windows;
+import org.apache.kafka.streams.state.SessionStore;
+import org.apache.kafka.test.KStreamTestDriver;
import org.apache.kafka.test.MockAggregator;
import org.apache.kafka.test.MockInitializer;
import org.apache.kafka.test.MockReducer;
+import org.apache.kafka.test.TestUtils;
import org.junit.Before;
import org.junit.Test;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
public class KGroupedStreamImplTest {
+ private static final String TOPIC = "topic";
+ private final KStreamBuilder builder = new KStreamBuilder();
private KGroupedStream<String, String> groupedStream;
@Before
public void before() {
- final KStream<String, String> stream = new KStreamBuilder().stream("topic");
- groupedStream = stream.groupByKey();
+ final KStream<String, String> stream = builder.stream(Serdes.String(), Serdes.String(), TOPIC);
+ groupedStream = stream.groupByKey(Serdes.String(), Serdes.String());
}
@Test(expected = NullPointerException.class)
@@ -63,7 +81,7 @@ public class KGroupedStreamImplTest {
@Test(expected = NullPointerException.class)
public void shouldNotHaveNullWindowsWithWindowedReduce() throws Exception {
- groupedStream.reduce(MockReducer.STRING_ADDER, null, "store");
+ groupedStream.reduce(MockReducer.STRING_ADDER, (Windows) null, "store");
}
@Test(expected = NullPointerException.class)
@@ -114,4 +132,209 @@ public class KGroupedStreamImplTest {
StateStoreSupplier storeSupplier = null;
groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.STRING_ADDER, TimeWindows.of(10), storeSupplier);
}
+
+ @Test
+ public void shouldAggregateSessionWindows() throws Exception {
+ final Map<Windowed<String>, Integer> results = new HashMap<>();
+ groupedStream.aggregate(new Initializer<Integer>() {
+ @Override
+ public Integer apply() {
+ return 0;
+ }
+ }, new Aggregator<String, String, Integer>() {
+ @Override
+ public Integer apply(final String aggKey, final String value, final Integer aggregate) {
+ return aggregate + 1;
+ }
+ }, new Merger<String, Integer>() {
+ @Override
+ public Integer apply(final String aggKey, final Integer aggOne, final Integer aggTwo) {
+ return aggOne + aggTwo;
+ }
+ }, SessionWindows.with(30), Serdes.Integer(), "session-store")
+ .foreach(new ForeachAction<Windowed<String>, Integer>() {
+ @Override
+ public void apply(final Windowed<String> key, final Integer value) {
+ results.put(key, value);
+ }
+ });
+
+ final KStreamTestDriver driver = new KStreamTestDriver(builder, TestUtils.tempDirectory());
+ driver.setTime(10);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(15);
+ driver.process(TOPIC, "2", "2");
+ driver.setTime(30);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(70);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(90);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(100);
+ driver.process(TOPIC, "1", "1");
+ driver.flushState();
+ assertEquals(Integer.valueOf(2), results.get(new Windowed<>("1", new TimeWindow(10, 30))));
+ assertEquals(Integer.valueOf(1), results.get(new Windowed<>("2", new TimeWindow(15, 15))));
+ assertEquals(Integer.valueOf(3), results.get(new Windowed<>("1", new TimeWindow(70, 100))));
+ }
+
+ @Test
+ public void shouldCountSessionWindows() throws Exception {
+ final Map<Windowed<String>, Long> results = new HashMap<>();
+ groupedStream.count(SessionWindows.with(30), "session-store")
+ .foreach(new ForeachAction<Windowed<String>, Long>() {
+ @Override
+ public void apply(final Windowed<String> key, final Long value) {
+ results.put(key, value);
+ }
+ });
+ final KStreamTestDriver driver = new KStreamTestDriver(builder, TestUtils.tempDirectory());
+ driver.setTime(10);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(15);
+ driver.process(TOPIC, "2", "2");
+ driver.setTime(30);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(70);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(90);
+ driver.process(TOPIC, "1", "1");
+ driver.setTime(100);
+ driver.process(TOPIC, "1", "1");
+ driver.flushState();
+ assertEquals(Long.valueOf(2), results.get(new Windowed<>("1", new TimeWindow(10, 30))));
+ assertEquals(Long.valueOf(1), results.get(new Windowed<>("2", new TimeWindow(15, 15))));
+ assertEquals(Long.valueOf(3), results.get(new Windowed<>("1", new TimeWindow(70, 100))));
+ }
+
+ @Test
+ public void shouldReduceSessionWindows() throws Exception {
+ final Map<Windowed<String>, String> results = new HashMap<>();
+ groupedStream.reduce(
+ new Reducer<String>() {
+ @Override
+ public String apply(final String value1, final String value2) {
+ return value1 + ":" + value2;
+ }
+ }, SessionWindows.with(30),
+ "session-store")
+ .foreach(new ForeachAction<Windowed<String>, String>() {
+ @Override
+ public void apply(final Windowed<String> key, final String value) {
+ results.put(key, value);
+ }
+ });
+ final KStreamTestDriver driver = new KStreamTestDriver(builder, TestUtils.tempDirectory());
+ driver.setTime(10);
+ driver.process(TOPIC, "1", "A");
+ driver.setTime(15);
+ driver.process(TOPIC, "2", "Z");
+ driver.setTime(30);
+ driver.process(TOPIC, "1", "B");
+ driver.setTime(70);
+ driver.process(TOPIC, "1", "A");
+ driver.setTime(90);
+ driver.process(TOPIC, "1", "B");
+ driver.setTime(100);
+ driver.process(TOPIC, "1", "C");
+ driver.flushState();
+ assertEquals("A:B", results.get(new Windowed<>("1", new TimeWindow(10, 30))));
+ assertEquals("Z", results.get(new Windowed<>("2", new TimeWindow(15, 15))));
+ assertEquals("A:B:C", results.get(new Windowed<>("1", new TimeWindow(70, 100))));
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullReducerWhenReducingSessionWindows() throws Exception {
+ groupedStream.reduce(null, SessionWindows.with(10), "store");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullSessionWindowsReducingSessionWindows() throws Exception {
+ groupedStream.reduce(MockReducer.STRING_ADDER, (SessionWindows) null, "store");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullStoreNameWhenReducingSessionWindows() throws Exception {
+ groupedStream.reduce(MockReducer.STRING_ADDER, SessionWindows.with(10), (String) null);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullStateStoreSupplierNameWhenReducingSessionWindows() throws Exception {
+ groupedStream.reduce(MockReducer.STRING_ADDER, SessionWindows.with(10), (StateStoreSupplier<SessionStore>) null);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullInitializerWhenAggregatingSessionWindows() throws Exception {
+ groupedStream.aggregate(null, MockAggregator.STRING_ADDER, new Merger<String, String>() {
+ @Override
+ public String apply(final String aggKey, final String aggOne, final String aggTwo) {
+ return null;
+ }
+ }, SessionWindows.with(10), Serdes.String(), "storeName");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullAggregatorWhenAggregatingSessionWindows() throws Exception {
+ groupedStream.aggregate(MockInitializer.STRING_INIT, null, new Merger<String, String>() {
+ @Override
+ public String apply(final String aggKey, final String aggOne, final String aggTwo) {
+ return null;
+ }
+ }, SessionWindows.with(10), Serdes.String(), "storeName");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullSessionMergerWhenAggregatingSessionWindows() throws Exception {
+ groupedStream.aggregate(MockInitializer.STRING_INIT,
+ MockAggregator.STRING_ADDER,
+ null,
+ SessionWindows.with(10),
+ Serdes.String(),
+ "storeName");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullSessionWindowsWhenAggregatingSessionWindows() throws Exception {
+ groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.STRING_ADDER, new Merger<String, String>() {
+ @Override
+ public String apply(final String aggKey, final String aggOne, final String aggTwo) {
+ return null;
+ }
+ }, null, Serdes.String(), "storeName");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullStoreNameWhenAggregatingSessionWindows() throws Exception {
+ groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.STRING_ADDER, new Merger<String, String>() {
+ @Override
+ public String apply(final String aggKey, final String aggOne, final String aggTwo) {
+ return null;
+ }
+ }, SessionWindows.with(10), Serdes.String(), (String) null);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullStateStoreSupplierNameWhenAggregatingSessionWindows() throws Exception {
+ groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.STRING_ADDER, new Merger<String, String>() {
+ @Override
+ public String apply(final String aggKey, final String aggOne, final String aggTwo) {
+ return null;
+ }
+ }, SessionWindows.with(10), Serdes.String(), (StateStoreSupplier<SessionStore>) null);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullSessionWindowsWhenCountingSessionWindows() throws Exception {
+ groupedStream.count((SessionWindows) null, "store");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullStoreNameWhenCountingSessionWindows() throws Exception {
+ groupedStream.count(SessionWindows.with(90), (String) null);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void shouldNotAcceptNullStoreStoreSupplierNameWhenCountingSessionWindows() throws Exception {
+ groupedStream.count(SessionWindows.with(90), (StateStoreSupplier<SessionStore>) null);
+ }
}
\ No newline at end of file
[4/5] kafka git commit: KAFKA-3452: Support session windows
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
index 922f4bc..f2c3b53 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/Stores.java
@@ -22,6 +22,7 @@ import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.state.internals.InMemoryKeyValueStoreSupplier;
import org.apache.kafka.streams.state.internals.InMemoryLRUCacheStoreSupplier;
import org.apache.kafka.streams.state.internals.RocksDBKeyValueStoreSupplier;
+import org.apache.kafka.streams.state.internals.RocksDBSessionStoreSupplier;
import org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,19 +106,28 @@ public class Stores {
private int numSegments = 0;
private long retentionPeriod = 0L;
private boolean retainDuplicates = false;
+ private boolean sessionWindows;
private boolean logged = true;
@Override
- public PersistentKeyValueFactory<K, V> windowed(final long windowSize, long retentionPeriod, int numSegments, boolean retainDuplicates) {
+ public PersistentKeyValueFactory<K, V> windowed(final long windowSize, final long retentionPeriod, final int numSegments, final boolean retainDuplicates) {
this.windowSize = windowSize;
this.numSegments = numSegments;
this.retentionPeriod = retentionPeriod;
this.retainDuplicates = retainDuplicates;
+ this.sessionWindows = false;
return this;
}
@Override
+ public PersistentKeyValueFactory<K, V> sessionWindowed(final long retentionPeriod) {
+ this.sessionWindows = true;
+ this.retentionPeriod = retentionPeriod;
+ return this;
+ }
+
+ @Override
public PersistentKeyValueFactory<K, V> enableLogging(final Map<String, String> config) {
logged = true;
logConfig.putAll(config);
@@ -140,7 +150,9 @@ public class Stores {
@Override
public StateStoreSupplier build() {
log.trace("Creating RocksDb Store name={} numSegments={} logged={}", name, numSegments, logged);
- if (numSegments > 0) {
+ if (sessionWindows) {
+ return new RocksDBSessionStoreSupplier<>(name, retentionPeriod, keySerde, valueSerde, logged, logConfig, cachingEnabled);
+ } else if (numSegments > 0) {
return new RocksDBWindowStoreSupplier<>(name, retentionPeriod, numSegments, retainDuplicates, keySerde, valueSerde, windowSize, logged, logConfig, cachingEnabled);
}
return new RocksDBKeyValueStoreSupplier<>(name, keySerde, valueSerde, logged, logConfig, cachingEnabled);
@@ -381,7 +393,6 @@ public class Stores {
/**
* Set the persistent store as a windowed key-value store
- *
* @param windowSize size of the windows
* @param retentionPeriod the maximum period of time in milli-second to keep each window in this store
* @param numSegments the maximum number of segments for rolling the windowed store
@@ -390,6 +401,12 @@ public class Stores {
PersistentKeyValueFactory<K, V> windowed(final long windowSize, long retentionPeriod, int numSegments, boolean retainDuplicates);
/**
+ * Set the persistent store as a {@link SessionStore} for use with {@link org.apache.kafka.streams.kstream.SessionWindows}
+ * @param retentionPeriod period of time in milliseconds to keep each window in this store
+ */
+ PersistentKeyValueFactory<K, V> sessionWindowed(final long retentionPeriod);
+
+ /**
* Indicates that a changelog should be created for the store. The changelog will be created
* with the provided cleanupPolicy and configs.
*
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
index cfe6bd3..6da40b7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingKeyValueStore.java
@@ -169,7 +169,7 @@ class CachingKeyValueStore<K, V> implements KeyValueStore<K, V>, CachedStateStor
validateStoreOpen();
final byte[] origFrom = serdes.rawKey(from);
final byte[] origTo = serdes.rawKey(to);
- final PeekingKeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>(underlying.range(Bytes.wrap(origFrom), Bytes.wrap(origTo)));
+ final KeyValueIterator<Bytes, byte[]> storeIterator = underlying.range(Bytes.wrap(origFrom), Bytes.wrap(origTo));
final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.range(name, origFrom, origTo);
return new MergedSortedCacheKeyValueStoreIterator<>(cacheIterator, storeIterator, serdes);
}
@@ -177,7 +177,7 @@ class CachingKeyValueStore<K, V> implements KeyValueStore<K, V>, CachedStateStor
@Override
public KeyValueIterator<K, V> all() {
validateStoreOpen();
- final PeekingKeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>(underlying.all());
+ final KeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>(name, underlying.all());
final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.all(name);
return new MergedSortedCacheKeyValueStoreIterator<>(cacheIterator, storeIterator, serdes);
}
@@ -223,7 +223,8 @@ class CachingKeyValueStore<K, V> implements KeyValueStore<K, V>, CachedStateStor
validateStoreOpen();
final byte[] rawKey = serdes.rawKey(key);
final V v = get(rawKey);
- put(rawKey, null);
+ cache.delete(name, serdes.rawKey(key));
+ underlying.delete(Bytes.wrap(rawKey));
return v;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java
new file mode 100644
index 0000000..a012c63
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingSessionStore.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.CacheFlushListener;
+import org.apache.kafka.streams.kstream.internals.SessionKeySerde;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
+import org.apache.kafka.streams.processor.internals.RecordContext;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.SessionStore;
+import org.apache.kafka.streams.state.StateSerdes;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+
+
+class CachingSessionStore<K, AGG> implements SessionStore<K, AGG>, CachedStateStore<Windowed<K>, AGG> {
+
+ private final SegmentedBytesStore bytesStore;
+ private final SessionKeySchema keySchema;
+ private Serde<K> keySerde;
+ private final Serde<AGG> aggSerde;
+ private InternalProcessorContext context;
+ private String name;
+ private StateSerdes<Windowed<K>, AGG> serdes;
+ private ThreadCache cache;
+ private CacheFlushListener<Windowed<K>, AGG> flushListener;
+
+ CachingSessionStore(final SegmentedBytesStore bytesStore,
+ final Serde<K> keySerde,
+ final Serde<AGG> aggSerde) {
+ this.bytesStore = bytesStore;
+ this.keySerde = keySerde;
+ this.aggSerde = aggSerde;
+ this.keySchema = new SessionKeySchema();
+ }
+
+ public KeyValueIterator<Windowed<K>, AGG> findSessionsToMerge(final K key,
+ final long earliestSessionEndTime,
+ final long latestSessionStartTime) {
+ validateStoreOpen();
+ final Bytes binarySessionId = Bytes.wrap(keySerde.serializer().serialize(name, key));
+ final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.range(name,
+ keySchema.lowerRange(binarySessionId,
+ earliestSessionEndTime).get(),
+ keySchema.upperRange(binarySessionId, latestSessionStartTime).get());
+ final KeyValueIterator<Bytes, byte[]> storeIterator = bytesStore.fetch(binarySessionId, earliestSessionEndTime, latestSessionStartTime);
+ final HasNextCondition hasNextCondition = keySchema.hasNextCondition(binarySessionId,
+ earliestSessionEndTime,
+ latestSessionStartTime);
+ final PeekingKeyValueIterator<Bytes, LRUCacheEntry> filteredCacheIterator = new FilteredCacheIterator(cacheIterator, hasNextCondition);
+ return new MergedSortedCacheKeyValueStoreIterator<>(filteredCacheIterator, storeIterator, serdes);
+ }
+
+
+ public void remove(final Windowed<K> sessionKey) {
+ validateStoreOpen();
+ put(sessionKey, null);
+ }
+
+ public void put(final Windowed<K> key, AGG value) {
+ validateStoreOpen();
+ final Bytes binaryKey = SessionKeySerde.toBinary(key, keySerde.serializer());
+ final LRUCacheEntry entry = new LRUCacheEntry(serdes.rawValue(value), true, context.offset(),
+ key.window().end(), context.partition(), context.topic());
+ cache.put(name, binaryKey.get(), entry);
+ }
+
+ @Override
+ public KeyValueIterator<Windowed<K>, AGG> fetch(final K key) {
+ return findSessionsToMerge(key, 0, Long.MAX_VALUE);
+ }
+
+
+ public String name() {
+ return bytesStore.name();
+ }
+
+ @SuppressWarnings("unchecked")
+ public void init(final ProcessorContext context, final StateStore root) {
+ bytesStore.init(context, root);
+ initInternal((InternalProcessorContext) context);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void initInternal(final InternalProcessorContext context) {
+ this.context = context;
+
+ if (keySerde == null) {
+ keySerde = (Serde<K>) context.keySerde();
+ }
+
+
+ this.serdes = (StateSerdes<Windowed<K>, AGG>) new StateSerdes<>(bytesStore.name(),
+ new SessionKeySerde<>(keySerde),
+ aggSerde == null ? context.valueSerde() : aggSerde);
+
+
+ this.name = context.taskId() + "-" + bytesStore.name();
+ this.cache = this.context.getCache();
+ cache.addDirtyEntryFlushListener(name, new ThreadCache.DirtyEntryFlushListener() {
+ @Override
+ public void apply(final List<ThreadCache.DirtyEntry> entries) {
+ for (ThreadCache.DirtyEntry entry : entries) {
+ putAndMaybeForward(entry, context);
+ }
+ }
+ });
+
+ }
+
+ private void putAndMaybeForward(final ThreadCache.DirtyEntry entry, final InternalProcessorContext context) {
+ final Bytes binaryKey = entry.key();
+ final RecordContext current = context.recordContext();
+ context.setRecordContext(entry.recordContext());
+ try {
+ if (flushListener != null) {
+ final Windowed<K> key = SessionKeySerde.from(binaryKey.get(), keySerde.deserializer());
+ final AGG newValue = serdes.valueFrom(entry.newValue());
+ final AGG oldValue = fetchPrevious(binaryKey);
+ if (!(newValue == null && oldValue == null)) {
+ flushListener.apply(key, newValue == null ? null : newValue, oldValue);
+ }
+
+ }
+ bytesStore.put(binaryKey, entry.newValue());
+ } finally {
+ context.setRecordContext(current);
+ }
+ }
+
+ private AGG fetchPrevious(final Bytes key) {
+ final byte[] bytes = bytesStore.get(key);
+ if (bytes == null) {
+ return null;
+ }
+ return serdes.valueFrom(bytes);
+ }
+
+
+ public void flush() {
+ cache.flush(name);
+ bytesStore.flush();
+ }
+
+ public void close() {
+ flush();
+ bytesStore.close();
+ cache.close(name);
+ }
+
+ public boolean persistent() {
+ return bytesStore.persistent();
+ }
+
+ public boolean isOpen() {
+ return bytesStore.isOpen();
+ }
+
+ public void setFlushListener(CacheFlushListener<Windowed<K>, AGG> flushListener) {
+ this.flushListener = flushListener;
+ }
+
+ private void validateStoreOpen() {
+ if (!isOpen()) {
+ throw new InvalidStateStoreException("Store " + this.name + " is currently closed");
+ }
+ }
+
+
+ private static class FilteredCacheIterator implements PeekingKeyValueIterator<Bytes, LRUCacheEntry> {
+ private final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator;
+ private final HasNextCondition hasNextCondition;
+
+ FilteredCacheIterator(final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator,
+ final HasNextCondition hasNextCondition) {
+ this.cacheIterator = cacheIterator;
+ this.hasNextCondition = hasNextCondition;
+ }
+
+ @Override
+ public void close() {
+ // no-op
+ }
+
+ @Override
+ public Bytes peekNextKey() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return cacheIterator.peekNextKey();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return hasNextCondition.hasNext(cacheIterator);
+ }
+
+ @Override
+ public KeyValue<Bytes, LRUCacheEntry> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return cacheIterator.next();
+
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public KeyValue<Bytes, LRUCacheEntry> peekNext() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return cacheIterator.peekNext();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
index 71856fa..bd252f1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java
@@ -26,6 +26,7 @@ import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.StateStore;
import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
import org.apache.kafka.streams.processor.internals.RecordContext;
+import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.StateSerdes;
import org.apache.kafka.streams.state.WindowStore;
import org.apache.kafka.streams.state.WindowStoreIterator;
@@ -34,7 +35,7 @@ import java.util.List;
class CachingWindowStore<K, V> implements WindowStore<K, V>, CachedStateStore<Windowed<K>, V> {
- private final WindowStore<Bytes, byte[]> underlying;
+ private final SegmentedBytesStore underlying;
private final Serde<K> keySerde;
private final Serde<V> valueSerde;
private CacheFlushListener<Windowed<K>, V> flushListener;
@@ -44,7 +45,7 @@ class CachingWindowStore<K, V> implements WindowStore<K, V>, CachedStateStore<Wi
private InternalProcessorContext context;
private StateSerdes<K, V> serdes;
- CachingWindowStore(final WindowStore<Bytes, byte[]> underlying,
+ CachingWindowStore(final SegmentedBytesStore underlying,
final Serde<K> keySerde,
final Serde<V> valueSerde,
final long windowSize) {
@@ -84,8 +85,8 @@ class CachingWindowStore<K, V> implements WindowStore<K, V>, CachedStateStore<Wi
final long timestamp = WindowStoreUtils.timestampFromBinaryKey(binaryKey);
final Windowed<K> windowedKey = new Windowed<>(WindowStoreUtils.keyFromBinaryKey(binaryKey, serdes),
new TimeWindow(timestamp, timestamp + windowSize));
- maybeForward(entry, key, timestamp, windowedKey, (InternalProcessorContext) context);
- underlying.put(key, entry.newValue(), timestamp);
+ maybeForward(entry, Bytes.wrap(binaryKey), windowedKey, (InternalProcessorContext) context);
+ underlying.put(Bytes.wrap(WindowStoreUtils.toBinaryKey(key, timestamp, 0, WindowStoreUtils.INNER_SERDES)), entry.newValue());
}
}
});
@@ -94,7 +95,6 @@ class CachingWindowStore<K, V> implements WindowStore<K, V>, CachedStateStore<Wi
private void maybeForward(final ThreadCache.DirtyEntry entry,
final Bytes key,
- final long timestamp,
final Windowed<K> windowedKey,
final InternalProcessorContext context) {
if (flushListener != null) {
@@ -102,7 +102,7 @@ class CachingWindowStore<K, V> implements WindowStore<K, V>, CachedStateStore<Wi
context.setRecordContext(entry.recordContext());
try {
flushListener.apply(windowedKey,
- serdes.valueFrom(entry.newValue()), fetchPrevious(key, timestamp));
+ serdes.valueFrom(entry.newValue()), fetchPrevious(key));
} finally {
context.setRecordContext(current);
}
@@ -158,18 +158,18 @@ class CachingWindowStore<K, V> implements WindowStore<K, V>, CachedStateStore<Wi
byte[] binaryFrom = WindowStoreUtils.toBinaryKey(key, timeFrom, 0, serdes);
byte[] binaryTo = WindowStoreUtils.toBinaryKey(key, timeTo, 0, serdes);
- final WindowStoreIterator<byte[]> underlyingIterator = underlying.fetch(Bytes.wrap(serdes.rawKey(key)), timeFrom, timeTo);
+ final KeyValueIterator<Bytes, byte[]> underlyingIterator = underlying.fetch(Bytes.wrap(serdes.rawKey(key)), timeFrom, timeTo);
final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.range(name, binaryFrom, binaryTo);
- return new MergedSortedCachedWindowStoreIterator<>(cacheIterator, new DelegatingPeekingWindowIterator<>(underlyingIterator), serdes);
+ return new MergedSortedCachedWindowStoreIterator<>(cacheIterator, new DelegatingPeekingKeyValueIterator<>(name, underlyingIterator), serdes);
}
- private V fetchPrevious(final Bytes key, final long timestamp) {
- try (final WindowStoreIterator<byte[]> iterator = underlying.fetch(key, timestamp, timestamp)) {
- if (!iterator.hasNext()) {
- return null;
- }
- return serdes.valueFrom(iterator.next().value);
+
+ private V fetchPrevious(final Bytes key) {
+ final byte[] result = underlying.get(key);
+ if (result == null) {
+ return null;
}
+ return serdes.valueFrom(result);
}
private void validateStoreOpen() {
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStore.java
new file mode 100644
index 0000000..14b8f17
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStore.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+/**
+ * Simple wrapper around a {@link SegmentedBytesStore} to support writing
+ * updates to a changelog
+ */
+class ChangeLoggingSegmentedBytesStore implements SegmentedBytesStore {
+
+ private final SegmentedBytesStore bytesStore;
+ private StoreChangeLogger<Bytes, byte[]> changeLogger;
+
+
+ ChangeLoggingSegmentedBytesStore(final SegmentedBytesStore bytesStore) {
+ this.bytesStore = bytesStore;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public KeyValueIterator<Bytes, byte[]> fetch(final Bytes key, final long from, final long to) {
+ return bytesStore.fetch(key, from, to);
+ }
+
+ @Override
+ public void remove(final Bytes key) {
+ bytesStore.remove(key);
+ changeLogger.logChange(key, null);
+ }
+
+ @Override
+ public void put(final Bytes key, final byte[] value) {
+ if (key != null) {
+ bytesStore.put(key, value);
+ changeLogger.logChange(key, value);
+ }
+ }
+
+ @Override
+ public byte[] get(final Bytes key) {
+ return bytesStore.get(key);
+ }
+
+ @Override
+ public String name() {
+ return bytesStore.name();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(final ProcessorContext context, final StateStore root) {
+ bytesStore.init(context, root);
+ changeLogger = new StoreChangeLogger<>(name(), context, WindowStoreUtils.INNER_SERDES);
+ }
+
+ @Override
+ public void flush() {
+ bytesStore.flush();
+ }
+
+ @Override
+ public void close() {
+ bytesStore.close();
+ }
+
+ @Override
+ public boolean persistent() {
+ return bytesStore.persistent();
+ }
+
+ @Override
+ public boolean isOpen() {
+ return bytesStore.isOpen();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStore.java
index 5c47419..b6b2031 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStore.java
@@ -75,7 +75,7 @@ public class CompositeReadOnlyKeyValueStore<K, V> implements ReadOnlyKeyValueSto
}
};
final List<ReadOnlyKeyValueStore<K, V>> stores = storeProvider.stores(storeName, storeType);
- return new CompositeKeyValueIterator(stores.iterator(), nextIteratorFunction);
+ return new DelegatingPeekingKeyValueIterator<>(storeName, new CompositeKeyValueIterator(stores.iterator(), nextIteratorFunction));
}
@Override
@@ -91,7 +91,7 @@ public class CompositeReadOnlyKeyValueStore<K, V> implements ReadOnlyKeyValueSto
}
};
final List<ReadOnlyKeyValueStore<K, V>> stores = storeProvider.stores(storeName, storeType);
- return new CompositeKeyValueIterator(stores.iterator(), nextIteratorFunction);
+ return new DelegatingPeekingKeyValueIterator<>(storeName, new CompositeKeyValueIterator(stores.iterator(), nextIteratorFunction));
}
@Override
@@ -132,6 +132,11 @@ public class CompositeReadOnlyKeyValueStore<K, V> implements ReadOnlyKeyValueSto
}
@Override
+ public K peekNextKey() {
+ throw new UnsupportedOperationException("peekNextKey not supported");
+ }
+
+ @Override
public boolean hasNext() {
while ((current == null || !current.hasNext())
&& storeIterator.hasNext()) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java
new file mode 100644
index 0000000..ea10b1a
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStore.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.QueryableStoreType;
+import org.apache.kafka.streams.state.ReadOnlySessionStore;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * Wrapper over the underlying {@link ReadOnlySessionStore}s found in a {@link
+ * org.apache.kafka.streams.processor.internals.ProcessorTopology}
+ */
+public class CompositeReadOnlySessionStore<K, V> implements ReadOnlySessionStore<K, V> {
+ private final StateStoreProvider storeProvider;
+ private final QueryableStoreType<ReadOnlySessionStore<K, V>> queryableStoreType;
+ private final String storeName;
+
+ public CompositeReadOnlySessionStore(final StateStoreProvider storeProvider,
+ final QueryableStoreType<ReadOnlySessionStore<K, V>> queryableStoreType,
+ final String storeName) {
+ this.storeProvider = storeProvider;
+ this.queryableStoreType = queryableStoreType;
+ this.storeName = storeName;
+ }
+
+
+ @Override
+ public KeyValueIterator<Windowed<K>, V> fetch(final K key) {
+ final List<ReadOnlySessionStore<K, V>> stores = storeProvider.stores(storeName, queryableStoreType);
+ for (final ReadOnlySessionStore<K, V> store : stores) {
+ try {
+ final KeyValueIterator<Windowed<K>, V> result = store.fetch(key);
+ if (!result.hasNext()) {
+ result.close();
+ } else {
+ return result;
+ }
+ } catch (final InvalidStateStoreException ise) {
+ throw new InvalidStateStoreException("State store [" + storeName + "] is not available anymore" +
+ " and may have been migrated to another instance; " +
+ "please re-discover its location from the state metadata.");
+ }
+ }
+ return new KeyValueIterator<Windowed<K>, V>() {
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Windowed<K> peekNextKey() {
+ throw new NoSuchElementException();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public KeyValue<Windowed<K>, V> next() {
+ throw new NoSuchElementException();
+ }
+
+ @Override
+ public void remove() {
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
index 38d5108..61cd950 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIterator.java
@@ -17,20 +17,24 @@
package org.apache.kafka.streams.state.internals;
import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
import org.apache.kafka.streams.state.KeyValueIterator;
import java.util.NoSuchElementException;
-class DelegatingPeekingKeyValueIterator<K, V> implements PeekingKeyValueIterator<K, V> {
+class DelegatingPeekingKeyValueIterator<K, V> implements KeyValueIterator<K, V> {
+ private final String storeName;
private final KeyValueIterator<K, V> underlying;
private KeyValue<K, V> next;
+ private volatile boolean open = true;
- public DelegatingPeekingKeyValueIterator(final KeyValueIterator<K, V> underlying) {
+ DelegatingPeekingKeyValueIterator(final String storeName, final KeyValueIterator<K, V> underlying) {
+ this.storeName = storeName;
this.underlying = underlying;
}
@Override
- public K peekNextKey() {
+ public synchronized K peekNextKey() {
if (!hasNext()) {
throw new NoSuchElementException();
}
@@ -38,12 +42,16 @@ class DelegatingPeekingKeyValueIterator<K, V> implements PeekingKeyValueIterator
}
@Override
- public void close() {
+ public synchronized void close() {
underlying.close();
+ open = false;
}
@Override
- public boolean hasNext() {
+ public synchronized boolean hasNext() {
+ if (!open) {
+ throw new InvalidStateStoreException(String.format("Store %s has closed", storeName));
+ }
if (next != null) {
return true;
}
@@ -57,7 +65,7 @@ class DelegatingPeekingKeyValueIterator<K, V> implements PeekingKeyValueIterator
}
@Override
- public KeyValue<K, V> next() {
+ public synchronized KeyValue<K, V> next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIterator.java
deleted file mode 100644
index 402a363..0000000
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIterator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.state.internals;
-
-import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.state.WindowStoreIterator;
-
-import java.util.NoSuchElementException;
-
-class DelegatingPeekingWindowIterator<V> implements PeekingWindowIterator<V> {
- private final WindowStoreIterator<V> underlying;
- private KeyValue<Long, V> next;
-
- public DelegatingPeekingWindowIterator(final WindowStoreIterator<V> underlying) {
- this.underlying = underlying;
- }
-
- @Override
- public KeyValue<Long, V> peekNext() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- return next;
- }
-
- @Override
- public void close() {
- underlying.close();
- }
-
- @Override
- public boolean hasNext() {
- if (next != null) {
- return true;
- }
-
- if (!underlying.hasNext()) {
- return false;
- }
-
- next = underlying.next();
- return true;
- }
-
- @Override
- public KeyValue<Long, V> next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- final KeyValue<Long, V> result = next;
- next = null;
- return result;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("remove not supported");
- }
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/HasNextCondition.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/HasNextCondition.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/HasNextCondition.java
new file mode 100644
index 0000000..5171ebd
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/HasNextCondition.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+interface HasNextCondition {
+ boolean hasNext(final KeyValueIterator<Bytes, ?> iterator);
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
index d09630d..e00f8ab 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreSupplier.java
@@ -155,13 +155,13 @@ public class InMemoryKeyValueStoreSupplier<K, V> extends AbstractStoreSupplier<K
@Override
public synchronized KeyValueIterator<K, V> range(K from, K to) {
- return new MemoryStoreIterator<>(this.map.subMap(from, true, to, false).entrySet().iterator());
+ return new DelegatingPeekingKeyValueIterator<>(name, new MemoryStoreIterator<>(this.map.subMap(from, true, to, false).entrySet().iterator()));
}
@Override
public synchronized KeyValueIterator<K, V> all() {
final TreeMap<K, V> copy = new TreeMap<>(this.map);
- return new MemoryStoreIterator<>(copy.entrySet().iterator());
+ return new DelegatingPeekingKeyValueIterator<>(name, new MemoryStoreIterator<>(copy.entrySet().iterator()));
}
@Override
@@ -206,6 +206,11 @@ public class InMemoryKeyValueStoreSupplier<K, V> extends AbstractStoreSupplier<K
public void close() {
}
+ @Override
+ public K peekNextKey() {
+ throw new UnsupportedOperationException("peekNextKey not supported on MemoryStoreIterator");
+ }
+
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
index e77c642..b7e47e2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/LRUCacheEntry.java
@@ -50,7 +50,7 @@ class LRUCacheEntry implements RecordContext {
8 + // timestamp
8 + // offset
4 + // partition
- topic.length();
+ (topic == null ? 0 : topic.length());
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
index b79ad7d..7666155 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MemoryNavigableLRUCache.java
@@ -41,7 +41,7 @@ public class MemoryNavigableLRUCache<K, V> extends MemoryLRUCache<K, V> {
@Override
public KeyValueIterator<K, V> range(K from, K to) {
final TreeMap<K, V> treeMap = toTreeMap();
- return new MemoryNavigableLRUCache.CacheIterator<>(treeMap.navigableKeySet().subSet(from, true, to, true).iterator(), treeMap);
+ return new DelegatingPeekingKeyValueIterator<>(name(), new MemoryNavigableLRUCache.CacheIterator<>(treeMap.navigableKeySet().subSet(from, true, to, true).iterator(), treeMap));
}
@Override
@@ -85,5 +85,10 @@ public class MemoryNavigableLRUCache<K, V> extends MemoryLRUCache<K, V> {
public void close() {
// do nothing
}
+
+ @Override
+ public K peekNextKey() {
+ throw new UnsupportedOperationException("peekNextKey not supported");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIterator.java
index cbcd0f4..c9a6866 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIterator.java
@@ -31,13 +31,13 @@ import java.util.NoSuchElementException;
* @param <V>
*/
class MergedSortedCacheKeyValueStoreIterator<K, V> implements KeyValueIterator<K, V> {
- private final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator;
- private final PeekingKeyValueIterator<Bytes, byte[]> storeIterator;
+ private final PeekingKeyValueIterator<Bytes, LRUCacheEntry> cacheIterator;
+ private final KeyValueIterator<Bytes, byte[]> storeIterator;
private final StateSerdes<K, V> serdes;
private final Comparator<byte[]> comparator = Bytes.BYTES_LEXICO_COMPARATOR;
- public MergedSortedCacheKeyValueStoreIterator(final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator,
- final PeekingKeyValueIterator<Bytes, byte[]> storeIterator,
+ public MergedSortedCacheKeyValueStoreIterator(final PeekingKeyValueIterator<Bytes, LRUCacheEntry> cacheIterator,
+ final KeyValueIterator<Bytes, byte[]> storeIterator,
final StateSerdes<K, V> serdes) {
this.cacheIterator = cacheIterator;
this.storeIterator = storeIterator;
@@ -48,9 +48,9 @@ class MergedSortedCacheKeyValueStoreIterator<K, V> implements KeyValueIterator<K
public boolean hasNext() {
while (cacheIterator.hasNext() && isDeletedCacheEntry(cacheIterator.peekNext())) {
if (storeIterator.hasNext()) {
- final byte[] storeKey = storeIterator.peekNextKey().get();
+ final Bytes storeKey = storeIterator.peekNextKey();
// advance the store iterator if the key is the same as the deleted cache key
- if (comparator.compare(storeKey, cacheIterator.peekNext().key) == 0) {
+ if (storeKey.equals(cacheIterator.peekNextKey())) {
storeIterator.next();
}
}
@@ -60,20 +60,74 @@ class MergedSortedCacheKeyValueStoreIterator<K, V> implements KeyValueIterator<K
return cacheIterator.hasNext() || storeIterator.hasNext();
}
- private boolean isDeletedCacheEntry(final KeyValue<byte[], LRUCacheEntry> nextFromCache) {
+
+ private boolean isDeletedCacheEntry(final KeyValue<Bytes, LRUCacheEntry> nextFromCache) {
return nextFromCache.value.value == null;
}
@Override
public KeyValue<K, V> next() {
+
+ return internalNext(new NextValueFunction<KeyValue<K, V>>() {
+ @Override
+ public KeyValue<K, V> apply(final byte[] cacheKey, final byte[] storeKey) {
+ if (cacheKey == null) {
+ return nextStoreValue();
+ }
+
+ if (storeKey == null) {
+ return nextCacheValue();
+ }
+
+ final int comparison = comparator.compare(cacheKey, storeKey);
+ if (comparison > 0) {
+ return nextStoreValue();
+ } else if (comparison < 0) {
+ return nextCacheValue();
+ } else {
+ storeIterator.next();
+ return nextCacheValue();
+ }
+ }
+ });
+ }
+
+ @Override
+ public K peekNextKey() {
+ return internalNext(new NextValueFunction<K>() {
+ @Override
+ public K apply(final byte[] cacheKey, final byte[] storeKey) {
+ if (cacheKey == null) {
+ return serdes.keyFrom(storeKey);
+ }
+
+ if (storeKey == null) {
+ return serdes.keyFrom(cacheKey);
+ }
+
+ final int comparison = comparator.compare(cacheKey, storeKey);
+ if (comparison > 0) {
+ return serdes.keyFrom(storeKey);
+ } else {
+ return serdes.keyFrom(cacheKey);
+ }
+ }
+ });
+ }
+
+ interface NextValueFunction<T> {
+ T apply(final byte[] cacheKey, final byte [] storeKey);
+ }
+
+ private <T> T internalNext(final NextValueFunction<T> nextValueFunction) {
if (!hasNext()) {
throw new NoSuchElementException();
}
byte[] nextCacheKey = null;
if (cacheIterator.hasNext()) {
- nextCacheKey = cacheIterator.peekNextKey();
+ nextCacheKey = cacheIterator.peekNextKey().get();
}
byte[] nextStoreKey = null;
@@ -81,29 +135,12 @@ class MergedSortedCacheKeyValueStoreIterator<K, V> implements KeyValueIterator<K
nextStoreKey = storeIterator.peekNextKey().get();
}
- if (nextCacheKey == null) {
- return nextStoreValue();
- }
-
- if (nextStoreKey == null) {
- return nextCacheValue();
- }
-
- final int comparison = comparator.compare(nextCacheKey, nextStoreKey);
- if (comparison > 0) {
- return nextStoreValue();
- } else if (comparison < 0) {
- return nextCacheValue();
- } else {
- storeIterator.next();
- return nextCacheValue();
- }
-
+ return nextValueFunction.apply(nextCacheKey, nextStoreKey);
}
private KeyValue<K, V> nextCacheValue() {
- final KeyValue<byte[], LRUCacheEntry> next = cacheIterator.next();
- return KeyValue.pair(serdes.keyFrom(next.key), serdes.valueFrom(next.value.value));
+ final KeyValue<Bytes, LRUCacheEntry> next = cacheIterator.next();
+ return KeyValue.pair(serdes.keyFrom(next.key.get()), serdes.valueFrom(next.value.value));
}
private KeyValue<K, V> nextStoreValue() {
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCachedWindowStoreIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCachedWindowStoreIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCachedWindowStoreIterator.java
index 68f147f..e210e73 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCachedWindowStoreIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MergedSortedCachedWindowStoreIterator.java
@@ -16,7 +16,9 @@
*/
package org.apache.kafka.streams.state.internals;
+import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.StateSerdes;
import org.apache.kafka.streams.state.WindowStoreIterator;
@@ -30,11 +32,11 @@ import java.util.NoSuchElementException;
*/
class MergedSortedCachedWindowStoreIterator<K, V> implements WindowStoreIterator<V> {
private final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator;
- private final PeekingWindowIterator<byte[]> storeIterator;
+ private final KeyValueIterator<Bytes, byte[]> storeIterator;
private final StateSerdes<K, V> serdes;
public MergedSortedCachedWindowStoreIterator(final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator,
- final PeekingWindowIterator<byte[]> storeIterator,
+ final KeyValueIterator<Bytes, byte[]> storeIterator,
final StateSerdes<K, V> serdes) {
this.cacheIterator = cacheIterator;
this.storeIterator = storeIterator;
@@ -55,16 +57,16 @@ class MergedSortedCachedWindowStoreIterator<K, V> implements WindowStoreIterator
Long nextCacheTimestamp = null;
if (cacheIterator.hasNext()) {
- nextCacheTimestamp = WindowStoreUtils.timestampFromBinaryKey(cacheIterator.peekNextKey());
+ nextCacheTimestamp = WindowStoreUtils.timestampFromBinaryKey(cacheIterator.peekNextKey().get());
}
Long nextStoreTimestamp = null;
if (storeIterator.hasNext()) {
- nextStoreTimestamp = storeIterator.peekNext().key;
+ nextStoreTimestamp = WindowStoreUtils.timestampFromBinaryKey(storeIterator.peekNextKey().get());
}
if (nextCacheTimestamp == null) {
- return nextStoreValue();
+ return nextStoreValue(nextStoreTimestamp);
}
if (nextStoreTimestamp == null) {
@@ -73,7 +75,7 @@ class MergedSortedCachedWindowStoreIterator<K, V> implements WindowStoreIterator
final int comparison = nextCacheTimestamp.compareTo(nextStoreTimestamp);
if (comparison > 0) {
- return nextStoreValue();
+ return nextStoreValue(nextStoreTimestamp);
} else if (comparison < 0) {
return nextCacheValue(nextCacheTimestamp);
} else {
@@ -83,13 +85,13 @@ class MergedSortedCachedWindowStoreIterator<K, V> implements WindowStoreIterator
}
private KeyValue<Long, V> nextCacheValue(final Long timestamp) {
- final KeyValue<byte[], LRUCacheEntry> next = cacheIterator.next();
+ final KeyValue<Bytes, LRUCacheEntry> next = cacheIterator.next();
return KeyValue.pair(timestamp, serdes.valueFrom(next.value.value));
}
- private KeyValue<Long, V> nextStoreValue() {
- final KeyValue<Long, byte[]> next = storeIterator.next();
- return KeyValue.pair(next.key, serdes.valueFrom(next.value));
+ private KeyValue<Long, V> nextStoreValue(final Long timestamp) {
+ final KeyValue<Bytes, byte[]> next = storeIterator.next();
+ return KeyValue.pair(timestamp, serdes.valueFrom(next.value));
}
@Override
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
index 2e16af2..fb8b3b1 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java
@@ -212,5 +212,10 @@ public class MeteredKeyValueStore<K, V> implements KeyValueStore<K, V> {
metrics.recordLatency(this.sensor, this.startNs, time.nanoseconds());
}
}
+
+ @Override
+ public K1 peekNextKey() {
+ return iter.peekNextKey();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStore.java
new file mode 100644
index 0000000..5181c08
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStore.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.common.utils.SystemTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsMetrics;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+class MeteredSegmentedBytesStore implements SegmentedBytesStore {
+
+ private final SegmentedBytesStore inner;
+ private final String metricScope;
+ private final Time time;
+
+ private Sensor putTime;
+ private Sensor fetchTime;
+ private Sensor flushTime;
+ private StreamsMetrics metrics;
+ private Sensor getTime;
+ private Sensor removeTime;
+
+ MeteredSegmentedBytesStore(final SegmentedBytesStore inner, String metricScope, Time time) {
+ this.inner = inner;
+ this.metricScope = metricScope;
+ this.time = time != null ? time : new SystemTime();
+ }
+
+ @Override
+ public String name() {
+ return inner.name();
+ }
+
+ @Override
+ public void init(ProcessorContext context, StateStore root) {
+ final String name = name();
+ this.metrics = context.metrics();
+ this.putTime = this.metrics.addLatencySensor(metricScope, name, "put");
+ this.fetchTime = this.metrics.addLatencySensor(metricScope, name, "fetch");
+ this.flushTime = this.metrics.addLatencySensor(metricScope, name, "flush");
+ this.getTime = this.metrics.addLatencySensor(metricScope, name, "get");
+ this.removeTime = this.metrics.addLatencySensor(metricScope, name, "remove");
+
+ final Sensor restoreTime = this.metrics.addLatencySensor(metricScope, name, "restore");
+ // register and possibly restore the state from the logs
+ final long startNs = time.nanoseconds();
+ try {
+ inner.init(context, root);
+ } finally {
+ this.metrics.recordLatency(restoreTime, startNs, time.nanoseconds());
+ }
+ }
+
+ @Override
+ public boolean persistent() {
+ return inner.persistent();
+ }
+
+ @Override
+ public boolean isOpen() {
+ return inner.isOpen();
+ }
+
+ @Override
+ public byte[] get(final Bytes key) {
+ final long startNs = time.nanoseconds();
+ try {
+ return inner.get(key);
+ } finally {
+ this.metrics.recordLatency(this.getTime, startNs, time.nanoseconds());
+ }
+ }
+
+ @Override
+ public KeyValueIterator<Bytes, byte[]> fetch(final Bytes key, long timeFrom, long timeTo) {
+ return new MeteredSegmentedBytesStoreIterator(inner.fetch(key, timeFrom, timeTo), this.fetchTime);
+ }
+
+ @Override
+ public void remove(final Bytes key) {
+ final long startNs = time.nanoseconds();
+ try {
+ inner.remove(key);
+ } finally {
+ this.metrics.recordLatency(this.removeTime, startNs, time.nanoseconds());
+ }
+ }
+
+ @Override
+ public void put(final Bytes key, final byte[] value) {
+ long startNs = time.nanoseconds();
+ try {
+ this.inner.put(key, value);
+ } finally {
+ this.metrics.recordLatency(this.putTime, startNs, time.nanoseconds());
+ }
+ }
+
+ @Override
+ public void close() {
+ inner.close();
+ }
+
+ @Override
+ public void flush() {
+ final long startNs = time.nanoseconds();
+ try {
+ this.inner.flush();
+ } finally {
+ this.metrics.recordLatency(this.flushTime, startNs, time.nanoseconds());
+ }
+ }
+
+ private class MeteredSegmentedBytesStoreIterator implements KeyValueIterator<Bytes, byte[]> {
+
+ private final KeyValueIterator<Bytes, byte[]> iter;
+ private final Sensor sensor;
+ private final long startNs;
+
+ MeteredSegmentedBytesStoreIterator(final KeyValueIterator<Bytes, byte[]> iter, Sensor sensor) {
+ this.iter = iter;
+ this.sensor = sensor;
+ this.startNs = time.nanoseconds();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public KeyValue<Bytes, byte[]> next() {
+ return iter.next();
+ }
+
+ @Override
+ public void remove() {
+ iter.remove();
+ }
+
+ @Override
+ public void close() {
+ try {
+ iter.close();
+ } finally {
+ metrics.recordLatency(this.sensor, this.startNs, time.nanoseconds());
+ }
+ }
+
+ @Override
+ public Bytes peekNextKey() {
+ return iter.peekNextKey();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
index 4272f2b..70f8676 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/NamedCache.java
@@ -110,6 +110,7 @@ class NamedCache {
}
final List<ThreadCache.DirtyEntry> entries = new ArrayList<>();
+ final List<Bytes> deleted = new ArrayList<>();
// evicted already been removed from the cache so add it to the list of
// flushed entries and remove from dirtyKeys.
@@ -125,10 +126,16 @@ class NamedCache {
}
entries.add(new ThreadCache.DirtyEntry(key, node.entry.value, node.entry));
node.entry.markClean();
+ if (node.entry.value == null) {
+ deleted.add(node.key);
+ }
}
// clear dirtyKeys before the listener is applied as it may be re-entrant.
dirtyKeys.clear();
listener.apply(entries);
+ for (Bytes key : deleted) {
+ delete(key);
+ }
}
@@ -264,7 +271,6 @@ class NamedCache {
copy.addAll(keySet);
return copy.iterator();
}
-
synchronized Iterator<Bytes> allKeys() {
return keySetIterator(cache.navigableKeySet());
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingKeyValueIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingKeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingKeyValueIterator.java
index 9a3a05c..496a424 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingKeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingKeyValueIterator.java
@@ -16,9 +16,10 @@
*/
package org.apache.kafka.streams.state.internals;
+import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.state.KeyValueIterator;
public interface PeekingKeyValueIterator<K, V> extends KeyValueIterator<K, V> {
- K peekNextKey();
+ KeyValue<K, V> peekNext();
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingWindowIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingWindowIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingWindowIterator.java
deleted file mode 100644
index c112169..0000000
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/PeekingWindowIterator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.state.internals;
-
-import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.state.WindowStoreIterator;
-
-public interface PeekingWindowIterator<V> extends WindowStoreIterator<V> {
-
- KeyValue<Long, V> peekNext();
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java
new file mode 100644
index 0000000..31956ba
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStore.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateRestoreCallback;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+
+import java.util.List;
+
+class RocksDBSegmentedBytesStore implements SegmentedBytesStore {
+
+ private final String name;
+ private final Segments segments;
+ private final KeySchema keySchema;
+ private ProcessorContext context;
+ private volatile boolean open;
+
+
+ RocksDBSegmentedBytesStore(final String name,
+ final long retention,
+ final int numSegments,
+ final KeySchema keySchema) {
+ this.name = name;
+ this.keySchema = keySchema;
+ this.segments = new Segments(name, retention, numSegments);
+ }
+
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public KeyValueIterator<Bytes, byte[]> fetch(final Bytes key, final long from, final long to) {
+ final List<Segment> searchSpace = keySchema.segmentsToSearch(segments, from, to);
+
+ final Bytes binaryFrom = keySchema.lowerRange(key, from);
+ final Bytes binaryTo = keySchema.upperRange(key, to);
+
+ return new SegmentIterator(
+ searchSpace.iterator(),
+ keySchema.hasNextCondition(key, from, to),
+ binaryFrom, binaryTo);
+ }
+
+ @Override
+ public void remove(final Bytes key) {
+ final Segment segment = segments.getSegmentForTimestamp(keySchema.segmentTimestamp(key));
+ if (segment == null) {
+ return;
+ }
+ segment.delete(key);
+ }
+
+ @Override
+ public void put(final Bytes key, final byte[] value) {
+ final long segmentId = segments.segmentId(keySchema.segmentTimestamp(key));
+ final Segment segment = segments.getOrCreateSegment(segmentId, context);
+ if (segment != null) {
+ segment.put(key, value);
+ }
+ }
+
+
+ @Override
+ public byte[] get(final Bytes key) {
+ final Segment segment = segments.getSegmentForTimestamp(keySchema.segmentTimestamp(key));
+ if (segment == null) {
+ return null;
+ }
+ return segment.get(key);
+ }
+
+ @Override
+ public String name() {
+ return name;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(ProcessorContext context, StateStore root) {
+ this.context = context;
+
+ segments.openExisting(context);
+
+ // register and possibly restore the state from the logs
+ context.register(root, false, new StateRestoreCallback() {
+ @Override
+ public void restore(byte[] key, byte[] value) {
+ put(Bytes.wrap(key), value);
+ }
+ });
+
+ flush();
+ open = true;
+ }
+
+ @Override
+ public void flush() {
+ segments.flush();
+ }
+
+ @Override
+ public void close() {
+ open = false;
+ segments.close();
+ }
+
+ @Override
+ public boolean persistent() {
+ return true;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return open;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java
new file mode 100644
index 0000000..73c825c
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStore.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.SessionKeySerde;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.processor.StateStore;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.SessionStore;
+import org.apache.kafka.streams.state.StateSerdes;
+
+
+class RocksDBSessionStore<K, AGG> implements SessionStore<K, AGG> {
+
+ private final Serde<K> keySerde;
+ private final Serde<AGG> aggSerde;
+ private final SegmentedBytesStore bytesStore;
+ private StateSerdes<K, AGG> serdes;
+
+
+ RocksDBSessionStore(final SegmentedBytesStore bytesStore,
+ final Serde<K> keySerde,
+ final Serde<AGG> aggSerde) {
+ this.keySerde = keySerde;
+ this.bytesStore = bytesStore;
+ this.aggSerde = aggSerde;
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public KeyValueIterator<Windowed<K>, AGG> findSessionsToMerge(final K key, final long earliestSessionEndTime, final long latestSessionStartTime) {
+ final KeyValueIterator<Bytes, byte[]> bytesIterator = bytesStore.fetch(Bytes.wrap(serdes.rawKey(key)), earliestSessionEndTime, latestSessionStartTime);
+ return new SessionStoreIterator(bytesIterator, serdes);
+ }
+
+
+ @Override
+ public void remove(final Windowed<K> key) {
+ bytesStore.remove(SessionKeySerde.toBinary(key, serdes.keySerializer()));
+ }
+
+ @Override
+ public void put(final Windowed<K> sessionKey, final AGG aggregate) {
+ bytesStore.put(SessionKeySerde.toBinary(sessionKey, serdes.keySerializer()), aggSerde.serializer().serialize(bytesStore.name(), aggregate));
+ }
+
+ @Override
+ public String name() {
+ return bytesStore.name();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(final ProcessorContext context, final StateStore root) {
+ this.serdes = new StateSerdes<>(bytesStore.name(),
+ keySerde == null ? (Serde<K>) context.keySerde() : keySerde,
+ aggSerde == null ? (Serde<AGG>) context.valueSerde() : aggSerde);
+
+ bytesStore.init(context, root);
+ }
+
+ @Override
+ public void flush() {
+ bytesStore.flush();
+ }
+
+ @Override
+ public void close() {
+ bytesStore.close();
+ }
+
+ @Override
+ public boolean persistent() {
+ return true;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return bytesStore.isOpen();
+ }
+
+ @Override
+ public KeyValueIterator<Windowed<K>, AGG> fetch(final K key) {
+ return findSessionsToMerge(key, 0, Long.MAX_VALUE);
+ }
+
+ private static class SessionStoreIterator<K, AGG> implements KeyValueIterator<Windowed<K>, AGG> {
+
+ private final KeyValueIterator<Bytes, byte[]> bytesIterator;
+ private final StateSerdes<K, AGG> serdes;
+
+ SessionStoreIterator(final KeyValueIterator<Bytes, byte[]> bytesIterator, final StateSerdes<K, AGG> serdes) {
+ this.bytesIterator = bytesIterator;
+ this.serdes = serdes;
+ }
+
+ @Override
+ public void close() {
+ bytesIterator.close();
+ }
+
+ @Override
+ public Windowed<K> peekNextKey() {
+ final Bytes bytes = bytesIterator.peekNextKey();
+ return SessionKeySerde.from(bytes.get(), serdes.keyDeserializer());
+ }
+
+ @Override
+ public boolean hasNext() {
+ return bytesIterator.hasNext();
+ }
+
+ @Override
+ public KeyValue<Windowed<K>, AGG> next() {
+ final KeyValue<Bytes, byte[]> next = bytesIterator.next();
+ return KeyValue.pair(SessionKeySerde.from(next.key.get(), serdes.keyDeserializer()), serdes.valueFrom(next.value));
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("remove not supported by SessionStoreIterator");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplier.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplier.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplier.java
new file mode 100644
index 0000000..7645472
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreSupplier.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.state.SessionStore;
+
+import java.util.Map;
+
+/**
+ * A {@link org.apache.kafka.streams.state.KeyValueStore} that stores all entries in a local RocksDB database.
+ *
+ * @param <K> the type of keys
+ * @param <V> the type of values
+ *
+ * @see org.apache.kafka.streams.state.Stores#create(String)
+ */
+
+public class RocksDBSessionStoreSupplier<K, V> extends AbstractStoreSupplier<K, V, SessionStore> implements WindowStoreSupplier<SessionStore> {
+
+ private static final int NUM_SEGMENTS = 3;
+ private final long retentionPeriod;
+ private final boolean enableCaching;
+
+ public RocksDBSessionStoreSupplier(String name, long retentionPeriod, Serde<K> keySerde, Serde<V> valueSerde, boolean logged, Map<String, String> logConfig, boolean enableCaching) {
+ super(name, keySerde, valueSerde, Time.SYSTEM, logged, logConfig);
+ this.retentionPeriod = retentionPeriod;
+ this.enableCaching = enableCaching;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public SessionStore<K, V> get() {
+ final RocksDBSegmentedBytesStore bytesStore = new RocksDBSegmentedBytesStore(name,
+ retentionPeriod,
+ NUM_SEGMENTS,
+ new SessionKeySchema());
+ final MeteredSegmentedBytesStore metered = new MeteredSegmentedBytesStore(logged ? new ChangeLoggingSegmentedBytesStore(bytesStore)
+ : bytesStore, "rocksdb-session-store", time);
+ if (enableCaching) {
+ return new CachingSessionStore<>(metered, keySerde, valueSerde);
+ }
+ return new RocksDBSessionStore<>(metered, keySerde, valueSerde);
+
+ }
+
+ public long retentionPeriod() {
+ return retentionPeriod;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 9c6703a..f838191 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -97,7 +97,6 @@ public class RocksDBStore<K, V> implements KeyValueStore<K, V> {
private StoreChangeLogger<Bytes, byte[]> changeLogger;
protected volatile boolean open = false;
- private ProcessorContext context;
public KeyValueStore<K, V> enableLogging() {
loggingEnabled = true;
@@ -130,6 +129,7 @@ public class RocksDBStore<K, V> implements KeyValueStore<K, V> {
options.setCreateIfMissing(true);
options.setErrorIfExists(false);
+
wOptions = new WriteOptions();
wOptions.setDisableWAL(true);
@@ -139,7 +139,6 @@ public class RocksDBStore<K, V> implements KeyValueStore<K, V> {
@SuppressWarnings("unchecked")
public void openDB(ProcessorContext context) {
- this.context = context;
final Map<String, Object> configs = context.appConfigs();
final Class<RocksDBConfigSetter> configSetterClass = (Class<RocksDBConfigSetter>) configs.get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG);
if (configSetterClass != null) {
@@ -242,11 +241,6 @@ public class RocksDBStore<K, V> implements KeyValueStore<K, V> {
}
}
- @SuppressWarnings("unchecked")
- synchronized void writeToStore(K key, V value) {
- putInternal(serdes.rawKey(key), serdes.rawValue(value));
- }
-
@Override
public synchronized V putIfAbsent(K key, V value) {
V originalValue = get(key);
@@ -453,6 +447,15 @@ public class RocksDBStore<K, V> implements KeyValueStore<K, V> {
iter.close();
}
+ @Override
+ public K peekNextKey() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return serdes.keyFrom(iter.key());
+
+ }
+
}
private class RocksDBRangeIterator extends RocksDbIterator {
[5/5] kafka git commit: KAFKA-3452: Support session windows
Posted by gu...@apache.org.
KAFKA-3452: Support session windows
Add support for SessionWindows based on design detailed in https://cwiki.apache.org/confluence/display/KAFKA/KIP-94+Session+Windows.
This includes refactoring of the RocksDBWindowStore such that functionality common with the RocksDBSessionStore isn't duplicated.
Author: Damian Guy <da...@gmail.com>
Reviewers: Eno Thereska <en...@gmail.com>, Matthias J. Sax <ma...@confluent.io>, Guozhang Wang <wa...@gmail.com>
Closes #2166 from dguy/kafka-3452-session-merge
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/e0de3a42
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/e0de3a42
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/e0de3a42
Branch: refs/heads/trunk
Commit: e0de3a4211a3701c98230b115fadfb67b655e3cf
Parents: 00964ec
Author: Damian Guy <da...@gmail.com>
Authored: Fri Jan 6 10:12:30 2017 -0800
Committer: Guozhang Wang <wa...@gmail.com>
Committed: Fri Jan 6 10:12:30 2017 -0800
----------------------------------------------------------------------
.../kafka/streams/kstream/KGroupedStream.java | 267 +++++++++++++
.../apache/kafka/streams/kstream/Merger.java | 40 ++
.../kafka/streams/kstream/SessionWindows.java | 105 ++++++
.../apache/kafka/streams/kstream/Window.java | 7 +
.../apache/kafka/streams/kstream/Windows.java | 2 +-
.../kafka/streams/kstream/internals/Change.java | 16 +
.../kstream/internals/KGroupedStreamImpl.java | 153 +++++++-
.../kstream/internals/KStreamAggregate.java | 4 +-
.../kstream/internals/KStreamReduce.java | 4 +-
.../KStreamSessionWindowAggregate.java | 165 ++++++++
.../internals/KStreamWindowAggregate.java | 6 +-
.../kstream/internals/KStreamWindowReduce.java | 6 +-
.../kstream/internals/KTableAggregate.java | 4 +-
.../streams/kstream/internals/KTableReduce.java | 4 +-
.../streams/kstream/internals/KTableSource.java | 4 +-
.../kstream/internals/SessionKeySerde.java | 149 ++++++++
.../kstream/internals/TupleForwarder.java | 21 +-
.../streams/processor/TopologyBuilder.java | 6 +-
.../kafka/streams/state/KeyValueIterator.java | 6 +
.../streams/state/QueryableStoreTypes.java | 20 +
.../streams/state/ReadOnlySessionStore.java | 40 ++
.../kafka/streams/state/SessionStore.java | 47 +++
.../org/apache/kafka/streams/state/Stores.java | 23 +-
.../state/internals/CachingKeyValueStore.java | 7 +-
.../state/internals/CachingSessionStore.java | 242 ++++++++++++
.../state/internals/CachingWindowStore.java | 28 +-
.../ChangeLoggingSegmentedBytesStore.java | 95 +++++
.../CompositeReadOnlyKeyValueStore.java | 9 +-
.../CompositeReadOnlySessionStore.java | 89 +++++
.../DelegatingPeekingKeyValueIterator.java | 20 +-
.../DelegatingPeekingWindowIterator.java | 73 ----
.../state/internals/HasNextCondition.java | 24 ++
.../InMemoryKeyValueStoreSupplier.java | 9 +-
.../streams/state/internals/LRUCacheEntry.java | 2 +-
.../internals/MemoryNavigableLRUCache.java | 7 +-
.../MergedSortedCacheKeyValueStoreIterator.java | 93 +++--
.../MergedSortedCachedWindowStoreIterator.java | 22 +-
.../state/internals/MeteredKeyValueStore.java | 5 +
.../internals/MeteredSegmentedBytesStore.java | 177 +++++++++
.../streams/state/internals/NamedCache.java | 8 +-
.../internals/PeekingKeyValueIterator.java | 3 +-
.../state/internals/PeekingWindowIterator.java | 25 --
.../internals/RocksDBSegmentedBytesStore.java | 134 +++++++
.../state/internals/RocksDBSessionStore.java | 143 +++++++
.../internals/RocksDBSessionStoreSupplier.java | 68 ++++
.../streams/state/internals/RocksDBStore.java | 17 +-
.../state/internals/RocksDBWindowStore.java | 374 +++----------------
.../internals/RocksDBWindowStoreSupplier.java | 19 +-
.../kafka/streams/state/internals/Segment.java | 42 +++
.../state/internals/SegmentIterator.java | 94 +++++
.../state/internals/SegmentedBytesStore.java | 122 ++++++
.../kafka/streams/state/internals/Segments.java | 179 +++++++++
.../state/internals/SessionKeySchema.java | 73 ++++
.../streams/state/internals/ThreadCache.java | 15 +-
.../state/internals/WindowStoreKeySchema.java | 59 +++
.../state/internals/WindowStoreSupplier.java | 24 ++
.../integration/JoinIntegrationTest.java | 2 +-
.../KStreamAggregationIntegrationTest.java | 203 ++++++++++
.../internals/KGroupedStreamImplTest.java | 229 +++++++++++-
...reamSessionWindowAggregateProcessorTest.java | 286 ++++++++++++++
.../kstream/internals/SessionKeySerdeTest.java | 87 +++++
.../internals/CachingSessionStoreTest.java | 204 ++++++++++
.../state/internals/CachingWindowStoreTest.java | 18 +-
.../ChangeLoggingSegmentedBytesStoreTest.java | 114 ++++++
.../CompositeReadOnlySessionStoreTest.java | 121 ++++++
.../DelegatingPeekingKeyValueIteratorTest.java | 11 +-
.../DelegatingPeekingWindowIteratorTest.java | 92 -----
.../state/internals/InMemoryKeyValueStore.java | 9 +-
...gedSortedCacheKeyValueStoreIteratorTest.java | 36 +-
...ergedSortedCacheWindowStoreIteratorTest.java | 44 +--
.../MeteredSegmentedBytesStoreTest.java | 122 ++++++
.../streams/state/internals/NamedCacheTest.java | 17 +
.../RocksDBSegmentedBytesStoreTest.java | 171 +++++++++
.../internals/RocksDBSessionStoreTest.java | 156 ++++++++
.../state/internals/RocksDBWindowStoreTest.java | 91 ++---
.../state/internals/SegmentIteratorTest.java | 148 ++++++++
.../streams/state/internals/SegmentsTest.java | 193 ++++++++++
.../state/internals/ThreadCacheTest.java | 17 +-
.../apache/kafka/test/KeyValueIteratorStub.java | 56 +++
.../apache/kafka/test/MockProcessorContext.java | 4 +
.../kafka/test/ReadOnlySessionStoreStub.java | 89 +++++
.../kafka/test/SegmentedBytesStoreStub.java | 95 +++++
82 files changed, 5257 insertions(+), 758 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
index 33a2791..fc2881a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java
@@ -21,6 +21,7 @@ import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.QueryableStoreType;
import org.apache.kafka.streams.state.WindowStore;
+import org.apache.kafka.streams.state.SessionStore;
/**
* {@link KGroupedStream} is an abstraction of a <i>grouped</i> record stream of key-value pairs.
@@ -179,6 +180,69 @@ public interface KGroupedStream<K, V> {
<W extends Window> KTable<Windowed<K>, Long> count(final Windows<W> windows,
final StateStoreSupplier<WindowStore> storeSupplier);
+
+ /**
+ * Count the number of records in this stream by the grouped key into {@link SessionWindows}.
+ * Records with {@code null} value are ignored.
+ * The result is written into a local {@link SessionStore} (which is basically an ever-updating
+ * materialized view) that can be queried using the provided {@code storeName}.
+ * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}).
+ * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where
+ * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID.
+ * <p>
+ * To query the local {@link SessionStore} it must be obtained via
+ * {@link org.apache.kafka.streams.KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+ * Use {@link StateStoreSupplier#name()} to get the store name:
+ * <pre>{@code
+ * KafkaStreams streams = ... // counting words
+ * String storeName = storeSupplier.name();
+ * ReadOnlySessionStore<String,Long> sessionStore = streams.store(storeName, QueryableStoreTypes.<String, Long>sessionStore());
+ * String key = "some-word";
+ * KeyValueIterator<Windowed<String>, Long> iterator = sessionStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+ * }</pre>
+ * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+ * query the value of the key on a parallel running instance of your Kafka Streams application.
+ *
+ *
+ * @param sessionWindows the specification of the aggregation {@link SessionWindows}
+ * @param storeName the name of the state store created from this operation.
+ * @return a windowed {@link KTable} which can be treated as a list of {@code KTable}s
+ * where each table contains records with unmodified keys and values
+ * that represent the latest (rolling) count (i.e., number of records) for each key within that window
+ */
+ KTable<Windowed<K>, Long> count(final SessionWindows sessionWindows, final String storeName);
+
+ /**
+ * Count the number of records in this stream by the grouped key into {@link SessionWindows}.
+ * Records with {@code null} value are ignored.
+ * The result is written into a local {@link SessionStore} provided by the given {@code storeSupplier}.
+ * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}).
+ * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where
+ * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID.
+ * <p>
+ * To query the local {@link SessionStore} it must be obtained via
+ * {@link org.apache.kafka.streams.KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+ * Use {@link StateStoreSupplier#name()} to get the store name:
+ * <pre>{@code
+ * KafkaStreams streams = ... // counting words
+ * String storeName = storeSupplier.name();
+ * ReadOnlySessionStore<String,Long> sessionStore = streams.store(storeName, QueryableStoreTypes.<String, Long>sessionStore());
+ * String key = "some-word";
+ * KeyValueIterator<Windowed<String>, Long> iterator = sessionStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+ * }</pre>
+ * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+ * query the value of the key on a parallel running instance of your Kafka Streams application.
+ *
+ *
+ * @param sessionWindows the specification of the aggregation {@link SessionWindows}
+ * @param storeSupplier user defined state store supplier {@link StateStoreSupplier}
+ * @return a windowed {@link KTable} which can be treated as a list of {@code KTable}s
+ * where each table contains records with unmodified keys and values
+ * that represent the latest (rolling) count (i.e., number of records) for each key within that window
+ */
+ KTable<Windowed<K>, Long> count(final SessionWindows sessionWindows,
+ final StateStoreSupplier<SessionStore> storeSupplier);
+
/**
* Combine the values of records in this stream by the grouped key.
* Records with {@code null} value are ignored.
@@ -360,6 +424,107 @@ public interface KGroupedStream<K, V> {
final Windows<W> windows,
final StateStoreSupplier<WindowStore> storeSupplier);
+ /**
+ * Combine values of this stream by the grouped key into {@link SessionWindows}.
+ * Combining implies that the type of the aggregate result is the same as the type of the input value
+ * (c.f. {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)}).
+ * The result is written into a local {@link SessionStore} (which is basically an ever-updating
+ * materialized view) that can be queried using the provided {@code storeName}.
+ * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}).
+ * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where
+ * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID.
+ * <p>
+ * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+ * the same key.
+ * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+ * parallel running Kafka Streams instances, and the cache size.
+ * You can configure the cache size via {@link org.apache.kafka.streams.StreamsConfig StreamsConfig} parameter
+ * {@link org.apache.kafka.streams.StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG CACHE_MAX_BYTES_BUFFERING_CONFIG}.
+ * <p>
+ * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+ * aggregate and the record's value.
+ * If there is no current aggregate the {@link Reducer} is not applied an the new aggregate will be the record's
+ * value as-is.
+ * Thus, {@link #reduce(Reducer, String)} can be used to compute aggregate functions like sum, min, or max.
+ * <p>
+ * To query the local {@link SessionStore} it must be obtained via
+ * {@link org.apache.kafka.streams.KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+ * <pre>{@code
+ * KafkaStreams streams = ... // compute sum
+ * ReadOnlySessionStore<String,Long> sessionStore = streams.store(storeName, QueryableStoreTypes.<String, Long>sessionStore());
+ * String key = "some-key";
+ * KeyValueIterator<Windowed<String>, Long> sumForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+ * }</pre>
+ * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+ * query the value of the key on a parallel running instance of your Kafka Streams application.
+ * <p>
+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+ * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+ * user-specified in {@link org.apache.kafka.streams.StreamsConfig StreamsConfig} via parameter
+ * {@link org.apache.kafka.streams.StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+ * provide {@code storeName}, and "-changelog" is a fixed suffix.
+ * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+ * @param reducer the instance of {@link Reducer}
+ * @param sessionWindows the specification of the aggregation {@link SessionWindows}
+ * @param storeName the name of the state store created from this operation
+ * @return a windowed {@link KTable} which can be treated as a list of {@code KTable}s
+ * where each table contains records with unmodified keys and values
+ * that represent the latest (rolling) aggregate for each key within that window
+ */
+ KTable<Windowed<K>, V> reduce(final Reducer<V> reducer,
+ final SessionWindows sessionWindows,
+ final String storeName);
+
+ /**
+ * Combine values of this stream by the grouped key into {@link SessionWindows}.
+ * Combining implies that the type of the aggregate result is the same as the type of the input value
+ * (c.f. {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)}).
+ * The result is written into a local {@link SessionStore} provided by the given {@code storeSupplier}.
+ * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}).
+ * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where
+ * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID.
+ * <p>
+ * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
+ * the same key.
+ * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
+ * parallel running Kafka Streams instances, and the cache size.
+ * You can configure the cache size via {@link org.apache.kafka.streams.StreamsConfig StreamsConfig} parameter
+ * {@link org.apache.kafka.streams.StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG CACHE_MAX_BYTES_BUFFERING_CONFIG}.
+ * <p>
+ * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
+ * aggregate and the record's value.
+ * If there is no current aggregate the {@link Reducer} is not applied an the new aggregate will be the record's
+ * value as-is.
+ * Thus, {@link #reduce(Reducer, String)} can be used to compute aggregate functions like sum, min, or max.
+ * <p>
+ * To query the local {@link SessionStore} it must be obtained via
+ * {@link org.apache.kafka.streams.KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+ * <pre>{@code
+ * KafkaStreams streams = ... // compute sum
+ * ReadOnlySessionStore<String,Long> sessionStore = streams.store(storeName, QueryableStoreTypes.<String, Long>sessionStore());
+ * String key = "some-key";
+ * KeyValueIterator<Windowed<String>, Long> sumForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+ * }</pre>
+ * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+ * query the value of the key on a parallel running instance of your Kafka Streams application.
+ * <p>
+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+ * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+ * user-specified in {@link org.apache.kafka.streams.StreamsConfig StreamsConfig} via parameter
+ * {@link org.apache.kafka.streams.StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+ * provide {@code storeName}, and "-changelog" is a fixed suffix.
+ * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+ * @param reducer the instance of {@link Reducer}
+ * @param sessionWindows the specification of the aggregation {@link SessionWindows}
+ * @param storeSupplier user defined state store supplier {@link StateStoreSupplier}
+ * @return a windowed {@link KTable} which can be treated as a list of {@code KTable}s
+ * where each table contains records with unmodified keys and values
+ * that represent the latest (rolling) aggregate for each key within that window
+ */
+ KTable<Windowed<K>, V> reduce(final Reducer<V> reducer,
+ final SessionWindows sessionWindows,
+ final StateStoreSupplier<SessionStore> storeSupplier);
+
/**
* Aggregate the values of records in this stream by the grouped key.
@@ -569,4 +734,106 @@ public interface KGroupedStream<K, V> {
final Windows<W> windows,
final StateStoreSupplier<WindowStore> storeSupplier);
+ /**
+ * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}.
+ * Records with {@code null} value are ignored.
+ * Aggregating is a generalization of {@link #reduce(Reducer, SessionWindows, String)} combining via
+ * reduce(...)} as it allows the result to have a different type than the input values.
+ * The result is written into a local {@link SessionStore} (which is basically an ever-updating
+ * materialized view) that can be queried using the provided {@code storeName}.
+ * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}).
+ * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where
+ * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID.
+ * <p>
+ * The specified {@link Initializer} is applied once per session directly before the first input record is
+ * processed to provide an initial intermediate aggregation result that is used to process the first record.
+ * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+ * aggregate (or for the very first record using the intermediate aggregation result provided via the
+ * {@link Initializer}) and the record's value.
+ * Thus, {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)} can be used to compute aggregate
+ * functions like count (c.f. {@link #count(String)})
+ * <p>
+ * To query the local {@link SessionStore} it must be obtained via
+ * {@link org.apache.kafka.streams.KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+ * Use {@link StateStoreSupplier#name()} to get the store name:
+ * <pre>{@code
+ * KafkaStreams streams = ... // some windowed aggregation on value type double
+ * Sting storeName = storeSupplier.name();
+ * ReadOnlySessionStore<String, Long> sessionStore = streams.store(storeName, QueryableStoreTypes.<String, Long>sessionStore());
+ * String key = "some-key";
+ * KeyValueIterator<Windowed<String>, Long> aggForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+ * }</pre>
+ * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+ * query the value of the key on a parallel running instance of your Kafka Streams application.
+ *
+ * @param initializer the instance of {@link Initializer}
+ * @param aggregator the instance of {@link Aggregator}
+ * @param sessionMerger the instance of {@link Merger}
+ * @param sessionWindows the specification of the aggregation {@link SessionWindows}
+ * @param aggValueSerde aggregate value serdes for materializing the aggregated table,
+ * if not specified the default serdes defined in the configs will be used
+ * @param <T> the value type of the resulting {@link KTable}
+ * @param storeName the name of the state store created from this operation
+ * @return a windowed {@link KTable} which can be treated as a list of {@code KTable}s
+ * where each table contains records with unmodified keys and values with type {@code T}
+ * that represent the latest (rolling) aggregate for each key within that window
+ */
+ <T> KTable<Windowed<K>, T> aggregate(final Initializer<T> initializer,
+ final Aggregator<? super K, ? super V, T> aggregator,
+ final Merger<? super K, T> sessionMerger,
+ final SessionWindows sessionWindows,
+ final Serde<T> aggValueSerde,
+ final String storeName);
+
+ /**
+ * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}.
+ * Records with {@code null} value are ignored.
+ * Aggregating is a generalization of {@link #reduce(Reducer, SessionWindows, String)} combining via
+ * reduce(...)} as it allows the result to have a different type than the input values.
+ * The result is written into a local {@link SessionStore} provided by the given {@code storeSupplier}.
+ * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}).
+ * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where
+ * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID.
+ * <p>
+ * The specified {@link Initializer} is applied once per session directly before the first input record is
+ * processed to provide an initial intermediate aggregation result that is used to process the first record.
+ * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
+ * aggregate (or for the very first record using the intermediate aggregation result provided via the
+ * {@link Initializer}) and the record's value.
+ * Thus, {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)} can be used to compute aggregate
+ * functions like count (c.f. {@link #count(String)})
+ * <p>
+ * To query the local {@link SessionStore} it must be obtained via
+ * {@link org.apache.kafka.streams.KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
+ * Use {@link StateStoreSupplier#name()} to get the store name:
+ * <pre>{@code
+ * KafkaStreams streams = ... // some windowed aggregation on value type double
+ * Sting storeName = storeSupplier.name();
+ * ReadOnlySessionStore<String, Long> sessionStore = streams.store(storeName, QueryableStoreTypes.<String, Long>sessionStore());
+ * String key = "some-key";
+ * KeyValueIterator<Windowed<String>, Long> aggForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
+ * }</pre>
+ * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+ * query the value of the key on a parallel running instance of your Kafka Streams application.
+ *
+ *
+ * @param initializer the instance of {@link Initializer}
+ * @param aggregator the instance of {@link Aggregator}
+ * @param sessionMerger the instance of {@link Merger}
+ * @param sessionWindows the specification of the aggregation {@link SessionWindows}
+ * @param aggValueSerde aggregate value serdes for materializing the aggregated table,
+ * if not specified the default serdes defined in the configs will be used
+ * @param storeSupplier user defined state store supplier {@link StateStoreSupplier}
+ * @param <T> the value type of the resulting {@link KTable}
+ * @return a windowed {@link KTable} which can be treated as a list of {@code KTable}s
+ * where each table contains records with unmodified keys and values with type {@code T}
+ * that represent the latest (rolling) aggregate for each key within that window
+ */
+ <T> KTable<Windowed<K>, T> aggregate(final Initializer<T> initializer,
+ final Aggregator<? super K, ? super V, T> aggregator,
+ final Merger<? super K, T> sessionMerger,
+ final SessionWindows sessionWindows,
+ final Serde<T> aggValueSerde,
+ final StateStoreSupplier<SessionStore> storeSupplier);
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/Merger.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Merger.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Merger.java
new file mode 100644
index 0000000..fcdb20f
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Merger.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream;
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ * The interface for merging aggregate values for {@link SessionWindows} with the given key.
+ *
+ * @param <K> key type
+ * @param <T> aggregate value type
+ */
+@InterfaceStability.Unstable
+public interface Merger<K, V> {
+
+ /**
+ * Compute a new aggregate from the key and two aggregates
+ *
+ * @param aggKey the key of the record
+ * @param aggOne the first aggregate
+ * @param aggTwo the second aggregate
+ * @return the new aggregate value
+ */
+ V apply(K aggKey, V aggOne, V aggTwo);
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
new file mode 100644
index 0000000..f9a399a
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindows.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream;
+
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ * A session based window specification used for aggregating events into sessions.
+ * <p>
+ * Sessions represent a period of activity separated by a defined gap of inactivity.
+ * Any events processed that fall within the inactivity gap of any existing sessions
+ * are merged into the existing sessions. If the event falls outside of the session gap
+ * then a new session will be created.
+ * <p>
+ * For example, If we have a session gap of 5 and the following data arrives:
+ * <pre>
+ * +--------------------------------------+
+ * | key | value | time |
+ * +-----------+-------------+------------+
+ * | A | 1 | 10 |
+ * +-----------+-------------+------------+
+ * | A | 2 | 12 |
+ * +-----------+-------------+------------+
+ * | A | 3 | 20 |
+ * +-----------+-------------+------------+
+ * </pre>
+ * <p>
+ * We'd have 2 sessions for key A. 1 starting from time 10 and ending at time 12 and another
+ * starting and ending at time 20. The length of the session is driven by the timestamps of
+ * the data within the session
+ * <p>
+ * If we then received another record:
+ * <p>
+ * <pre>
+ * +--------------------------------------+
+ * | key | value | time |
+ * +-----------+-------------+------------+
+ * | A | 4 | 16 |
+ * +-----------+-------------+------------+
+ * </pre>
+ * <p>
+ * The previous 2 sessions would be merged into a single session with start time 10 and end time 20.
+ * The aggregate value for this session would be the result of aggregating all 4 values.
+ */
+@InterfaceStability.Unstable
+public class SessionWindows {
+
+ private final long gapMs;
+ private long maintainDurationMs;
+
+ private SessionWindows(final long gapMs, final long maintainDurationMs) {
+ this.gapMs = gapMs;
+ this.maintainDurationMs = maintainDurationMs;
+ }
+
+ /**
+ * Create a new SessionWindows with the specified inactivity gap
+ * @param inactivityGapMs the gap of inactivity between sessions
+ * @return a new SessionWindows with the provided inactivity gap
+ * and default maintain duration
+ */
+ public static SessionWindows with(final long inactivityGapMs) {
+ return new SessionWindows(inactivityGapMs, Windows.DEFAULT_MAINTAIN_DURATION);
+ }
+
+ /**
+ * Set the window maintain duration in milliseconds of streams time.
+ * This retention time is a guaranteed <i>lower bound</i> for how long a window will be maintained.
+ *
+ * @return itself
+ */
+ public SessionWindows until(final long durationMs) {
+ this.maintainDurationMs = durationMs;
+ return this;
+ }
+
+ /**
+ * @return the inactivityGap
+ */
+ public long inactivityGap() {
+ return gapMs;
+ }
+
+ /**
+ * @return the minimum amount of time a window will be maintained for.
+ */
+ public long maintainMs() {
+ return maintainDurationMs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
index e1ea9a0..7d78d74 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Window.java
@@ -80,4 +80,11 @@ public abstract class Window {
return (int) (n % 0xFFFFFFFFL);
}
+ @Override
+ public String toString() {
+ return "Window{" +
+ "start=" + start +
+ ", end=" + end +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
index dbc52a8..ebd92fe 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Windows.java
@@ -28,7 +28,7 @@ public abstract class Windows<W extends Window> {
private static final int DEFAULT_NUM_SEGMENTS = 3;
- private static final long DEFAULT_MAINTAIN_DURATION = 24 * 60 * 60 * 1000L; // one day
+ static final long DEFAULT_MAINTAIN_DURATION = 24 * 60 * 60 * 1000L; // one day
private long maintainDurationMs;
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
index d7b868e..0946b6b 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/Change.java
@@ -17,6 +17,8 @@
package org.apache.kafka.streams.kstream.internals;
+import java.util.Objects;
+
public class Change<T> {
public final T newValue;
@@ -31,4 +33,18 @@ public class Change<T> {
public String toString() {
return "(" + newValue + "<-" + oldValue + ")";
}
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ final Change<?> change = (Change<?>) o;
+ return Objects.equals(newValue, change.newValue) &&
+ Objects.equals(oldValue, change.oldValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(newValue, oldValue);
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
index e50b6dc..a4fc793 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java
@@ -4,9 +4,9 @@
* copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
@@ -16,24 +16,27 @@ package org.apache.kafka.streams.kstream.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.kstream.Aggregator;
+import org.apache.kafka.streams.kstream.Initializer;
import org.apache.kafka.streams.kstream.KGroupedStream;
+import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.streams.kstream.KStreamBuilder;
+import org.apache.kafka.streams.kstream.Merger;
import org.apache.kafka.streams.kstream.Reducer;
-import org.apache.kafka.streams.kstream.Windowed;
-import org.apache.kafka.streams.kstream.KTable;
+import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.Window;
+import org.apache.kafka.streams.kstream.Windowed;
import org.apache.kafka.streams.kstream.Windows;
-import org.apache.kafka.streams.kstream.Initializer;
-import org.apache.kafka.streams.kstream.Aggregator;
import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.WindowStore;
+import org.apache.kafka.streams.state.SessionStore;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
-public class KGroupedStreamImpl<K, V> extends AbstractStream<K> implements KGroupedStream<K, V> {
+class KGroupedStreamImpl<K, V> extends AbstractStream<K> implements KGroupedStream<K, V> {
private static final String REDUCE_NAME = "KSTREAM-REDUCE-";
private static final String AGGREGATE_NAME = "KSTREAM-AGGREGATE-";
@@ -42,12 +45,12 @@ public class KGroupedStreamImpl<K, V> extends AbstractStream<K> implements KGrou
private final Serde<V> valSerde;
private final boolean repartitionRequired;
- public KGroupedStreamImpl(final KStreamBuilder topology,
- final String name,
- final Set<String> sourceNodes,
- final Serde<K> keySerde,
- final Serde<V> valSerde,
- final boolean repartitionRequired) {
+ KGroupedStreamImpl(final KStreamBuilder topology,
+ final String name,
+ final Set<String> sourceNodes,
+ final Serde<K> keySerde,
+ final Serde<V> valSerde,
+ final boolean repartitionRequired) {
super(topology, name, sourceNodes);
this.keySerde = keySerde;
this.valSerde = valSerde;
@@ -188,6 +191,130 @@ public class KGroupedStreamImpl<K, V> extends AbstractStream<K> implements KGrou
storeSupplier);
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> KTable<Windowed<K>, T> aggregate(final Initializer<T> initializer,
+ final Aggregator<? super K, ? super V, T> aggregator,
+ final Merger<? super K, T> sessionMerger,
+ final SessionWindows sessionWindows,
+ final Serde<T> aggValueSerde,
+ final String storeName) {
+ Objects.requireNonNull(storeName, "storeName can't be null");
+ return aggregate(initializer,
+ aggregator,
+ sessionMerger,
+ sessionWindows,
+ aggValueSerde,
+ storeFactory(keySerde, aggValueSerde, storeName)
+ .sessionWindowed(sessionWindows.maintainMs()).build());
+
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> KTable<Windowed<K>, T> aggregate(final Initializer<T> initializer,
+ final Aggregator<? super K, ? super V, T> aggregator,
+ final Merger<? super K, T> sessionMerger,
+ final SessionWindows sessionWindows,
+ final Serde<T> aggValueSerde,
+ final StateStoreSupplier<SessionStore> storeSupplier) {
+ Objects.requireNonNull(initializer, "initializer can't be null");
+ Objects.requireNonNull(aggregator, "aggregator can't be null");
+ Objects.requireNonNull(sessionWindows, "sessionWindows can't be null");
+ Objects.requireNonNull(sessionMerger, "sessionMerger can't be null");
+ Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
+
+ return (KTable<Windowed<K>, T>) doAggregate(
+ new KStreamSessionWindowAggregate<>(sessionWindows, storeSupplier.name(), initializer, aggregator, sessionMerger),
+ AGGREGATE_NAME,
+ storeSupplier);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public KTable<Windowed<K>, Long> count(final SessionWindows sessionWindows, final String storeName) {
+ Objects.requireNonNull(storeName, "storeName can't be null");
+ return count(sessionWindows,
+ storeFactory(keySerde, Serdes.Long(), storeName)
+ .sessionWindowed(sessionWindows.maintainMs()).build());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public KTable<Windowed<K>, Long> count(final SessionWindows sessionWindows,
+ final StateStoreSupplier<SessionStore> storeSupplier) {
+ Objects.requireNonNull(sessionWindows, "sessionWindows can't be null");
+ Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
+ final Initializer<Long> initializer = new Initializer<Long>() {
+ @Override
+ public Long apply() {
+ return 0L;
+ }
+ };
+ final Aggregator<K, V, Long> aggregator = new Aggregator<K, V, Long>() {
+ @Override
+ public Long apply(final K aggKey, final V value, final Long aggregate) {
+ return aggregate + 1;
+ }
+ };
+ final Merger<K, Long> sessionMerger = new Merger<K, Long>() {
+ @Override
+ public Long apply(final K aggKey, final Long aggOne, final Long aggTwo) {
+ return aggOne + aggTwo;
+ }
+ };
+
+ return aggregate(initializer, aggregator, sessionMerger, sessionWindows, Serdes.Long(), storeSupplier);
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public KTable<Windowed<K>, V> reduce(final Reducer<V> reducer,
+ final SessionWindows sessionWindows,
+ final String storeName) {
+
+ Objects.requireNonNull(storeName, "storeName can't be null");
+ return reduce(reducer, sessionWindows,
+ storeFactory(keySerde, valSerde, storeName)
+ .sessionWindowed(sessionWindows.maintainMs()).build());
+ }
+
+ @Override
+ public KTable<Windowed<K>, V> reduce(final Reducer<V> reducer,
+ final SessionWindows sessionWindows,
+ final StateStoreSupplier<SessionStore> storeSupplier) {
+ Objects.requireNonNull(reducer, "reducer can't be null");
+ Objects.requireNonNull(sessionWindows, "sessionWindows can't be null");
+ Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
+
+ final Initializer<V> initializer = new Initializer<V>() {
+ @Override
+ public V apply() {
+ return null;
+ }
+ };
+
+ final Aggregator<K, V, V> aggregator = new Aggregator<K, V, V>() {
+ @Override
+ public V apply(final K aggKey, final V value, final V aggregate) {
+ if (aggregate == null) {
+ return value;
+ }
+ return reducer.apply(aggregate, value);
+ }
+ };
+
+ final Merger<K, V> sessionMerger = new Merger<K, V>() {
+ @Override
+ public V apply(final K aggKey, final V aggOne, final V aggTwo) {
+ return aggregator.apply(aggKey, aggTwo, aggOne);
+ }
+ };
+
+ return aggregate(initializer, aggregator, sessionMerger, sessionWindows, valSerde, storeSupplier);
+ }
private <T> KTable<K, T> doAggregate(
final KStreamAggProcessorSupplier<K, ?, V, T> aggregateSupplier,
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
index 5bbda1f..fdc14df 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamAggregate.java
@@ -59,7 +59,7 @@ public class KStreamAggregate<K, V, T> implements KStreamAggProcessorSupplier<K,
public void init(ProcessorContext context) {
super.init(context);
store = (KeyValueStore<K, T>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues), sendOldValues);
}
@@ -82,7 +82,7 @@ public class KStreamAggregate<K, V, T> implements KStreamAggProcessorSupplier<K,
// update the store with the new value
store.put(key, newAgg);
- tupleForwarder.maybeForward(key, newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(key, newAgg, oldAgg);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamReduce.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamReduce.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamReduce.java
index 9af4368..fa26a7e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamReduce.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamReduce.java
@@ -56,7 +56,7 @@ public class KStreamReduce<K, V> implements KStreamAggProcessorSupplier<K, K, V,
super.init(context);
store = (KeyValueStore<K, V>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues), sendOldValues);
}
@@ -79,7 +79,7 @@ public class KStreamReduce<K, V> implements KStreamAggProcessorSupplier<K, K, V,
}
// update the store with the new value
store.put(key, newAgg);
- tupleForwarder.maybeForward(key, newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(key, newAgg, oldAgg);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
new file mode 100644
index 0000000..1af1f48
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregate.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.ProcessorStateException;
+import org.apache.kafka.streams.kstream.Aggregator;
+import org.apache.kafka.streams.kstream.Initializer;
+import org.apache.kafka.streams.kstream.Merger;
+import org.apache.kafka.streams.kstream.SessionWindows;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.processor.AbstractProcessor;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.processor.ProcessorContext;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.SessionStore;
+
+import java.util.ArrayList;
+import java.util.List;
+
+class KStreamSessionWindowAggregate<K, V, T> implements KStreamAggProcessorSupplier<K, Windowed<K>, V, T> {
+
+ private final String storeName;
+ private final SessionWindows windows;
+ private final Initializer<T> initializer;
+ private final Aggregator<? super K, ? super V, T> aggregator;
+ private final Merger<? super K, T> sessionMerger;
+
+ private boolean sendOldValues = false;
+
+ KStreamSessionWindowAggregate(final SessionWindows windows,
+ final String storeName,
+ final Initializer<T> initializer,
+ final Aggregator<? super K, ? super V, T> aggregator,
+ final Merger<? super K, T> sessionMerger) {
+ this.windows = windows;
+ this.storeName = storeName;
+ this.initializer = initializer;
+ this.aggregator = aggregator;
+ this.sessionMerger = sessionMerger;
+ }
+
+ @Override
+ public Processor<K, V> get() {
+ return new KStreamSessionWindowAggregateProcessor();
+ }
+
+ @Override
+ public void enableSendingOldValues() {
+ sendOldValues = true;
+ }
+
+ private class KStreamSessionWindowAggregateProcessor extends AbstractProcessor<K, V> {
+
+ private SessionStore<K, T> store;
+ private TupleForwarder<Windowed<K>, T> tupleForwarder;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void init(ProcessorContext context) {
+ super.init(context);
+ store = (SessionStore<K, T>) context.getStateStore(storeName);
+ tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues), sendOldValues);
+ }
+
+ @Override
+ public void process(final K key, final V value) {
+ // if the key is null, we do not need proceed aggregating
+ // the record with the table
+ if (key == null) {
+ return;
+ }
+
+ final long timestamp = context().timestamp();
+ final List<KeyValue<Windowed<K>, T>> merged = new ArrayList<>();
+ final TimeWindow newTimeWindow = new TimeWindow(timestamp, timestamp);
+ TimeWindow mergedWindow = newTimeWindow;
+ T agg = initializer.apply();
+
+ try (final KeyValueIterator<Windowed<K>, T> iterator = store.findSessionsToMerge(key, timestamp - windows.inactivityGap(),
+ timestamp + windows.inactivityGap())) {
+ while (iterator.hasNext()) {
+ final KeyValue<Windowed<K>, T> next = iterator.next();
+ merged.add(next);
+ agg = sessionMerger.apply(key, agg, next.value);
+ mergedWindow = mergeTimeWindow(mergedWindow, (TimeWindow) next.key.window());
+ }
+ }
+
+ agg = aggregator.apply(key, value, agg);
+ final Windowed<K> sessionKey = new Windowed<>(key, mergedWindow);
+ if (!mergedWindow.equals(newTimeWindow)) {
+ for (final KeyValue<Windowed<K>, T> session : merged) {
+ store.remove(session.key);
+ tupleForwarder.maybeForward(session.key, null, session.value);
+ }
+ }
+ store.put(sessionKey, agg);
+ tupleForwarder.maybeForward(sessionKey, agg, null);
+ }
+
+ }
+
+
+ private TimeWindow mergeTimeWindow(final TimeWindow one, final TimeWindow two) {
+ final long start = one.start() < two.start() ? one.start() : two.start();
+ final long end = one.end() > two.end() ? one.end() : two.end();
+ return new TimeWindow(start, end);
+ }
+
+ @Override
+ public KTableValueGetterSupplier<Windowed<K>, T> view() {
+ return new KTableValueGetterSupplier<Windowed<K>, T>() {
+ @Override
+ public KTableValueGetter<Windowed<K>, T> get() {
+ return new KTableSessionWindowValueGetter();
+ }
+
+ @Override
+ public String[] storeNames() {
+ return new String[] {storeName};
+ }
+ };
+ }
+
+ private class KTableSessionWindowValueGetter implements KTableValueGetter<Windowed<K>, T> {
+ private SessionStore<K, T> store;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void init(final ProcessorContext context) {
+ store = (SessionStore<K, T>) context.getStateStore(storeName);
+ }
+
+ @Override
+ public T get(final Windowed<K> key) {
+ try (KeyValueIterator<Windowed<K>, T> iter = store.findSessionsToMerge(key.key(), key.window().end(), key.window().end())) {
+ if (!iter.hasNext()) {
+ return null;
+ }
+ final T value = iter.next().value;
+ if (iter.hasNext()) {
+ throw new ProcessorStateException(String.format("Iterator for key [%s] on session store has more than one value", key));
+ }
+ return value;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
index d74a399..25fdda9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregate.java
@@ -68,7 +68,7 @@ public class KStreamWindowAggregate<K, V, T, W extends Window> implements KStrea
super.init(context);
windowStore = (WindowStore<K, T>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<>(windowStore, context, new ForwardingCacheFlushListener<Windowed<K>, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<>(windowStore, context, new ForwardingCacheFlushListener<Windowed<K>, V>(context, sendOldValues), sendOldValues);
}
@Override
@@ -110,7 +110,7 @@ public class KStreamWindowAggregate<K, V, T, W extends Window> implements KStrea
// update the store with the new value
windowStore.put(key, newAgg, window.start());
- tupleForwarder.maybeForward(new Windowed<>(key, window), newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(new Windowed<>(key, window), newAgg, oldAgg);
matchedWindows.remove(entry.key);
}
}
@@ -121,7 +121,7 @@ public class KStreamWindowAggregate<K, V, T, W extends Window> implements KStrea
T oldAgg = initializer.apply();
T newAgg = aggregator.apply(key, value, oldAgg);
windowStore.put(key, newAgg, windowStartMs);
- tupleForwarder.maybeForward(new Windowed<>(key, matchedWindows.get(windowStartMs)), newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(new Windowed<>(key, matchedWindows.get(windowStartMs)), newAgg, oldAgg);
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
index 5ee02e9..3eca401 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamWindowReduce.java
@@ -64,7 +64,7 @@ public class KStreamWindowReduce<K, V, W extends Window> implements KStreamAggPr
public void init(ProcessorContext context) {
super.init(context);
windowStore = (WindowStore<K, V>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<>(windowStore, context, new ForwardingCacheFlushListener<Windowed<K>, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<>(windowStore, context, new ForwardingCacheFlushListener<Windowed<K>, V>(context, sendOldValues), sendOldValues);
}
@Override
@@ -108,7 +108,7 @@ public class KStreamWindowReduce<K, V, W extends Window> implements KStreamAggPr
// update the store with the new value
windowStore.put(key, newAgg, window.start());
- tupleForwarder.maybeForward(new Windowed<>(key, window), newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(new Windowed<>(key, window), newAgg, oldAgg);
matchedWindows.remove(entry.key);
}
}
@@ -117,7 +117,7 @@ public class KStreamWindowReduce<K, V, W extends Window> implements KStreamAggPr
// create the new window for the rest of unmatched window that do not exist yet
for (long windowStartMs : matchedWindows.keySet()) {
windowStore.put(key, value, windowStartMs);
- tupleForwarder.maybeForward(new Windowed<>(key, matchedWindows.get(windowStartMs)), value, null, false);
+ tupleForwarder.maybeForward(new Windowed<>(key, matchedWindows.get(windowStartMs)), value, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableAggregate.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableAggregate.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableAggregate.java
index fd04fb3..a414200 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableAggregate.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableAggregate.java
@@ -61,7 +61,7 @@ public class KTableAggregate<K, V, T> implements KTableProcessorSupplier<K, V, T
public void init(final ProcessorContext context) {
super.init(context);
store = (KeyValueStore<K, T>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues), sendOldValues);
}
/**
@@ -92,7 +92,7 @@ public class KTableAggregate<K, V, T> implements KTableProcessorSupplier<K, V, T
// update the store with the new value
store.put(key, newAgg);
- tupleForwarder.maybeForward(key, newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(key, newAgg, oldAgg);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java
index 7b29d1c..30b314a 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableReduce.java
@@ -58,7 +58,7 @@ public class KTableReduce<K, V> implements KTableProcessorSupplier<K, V, V> {
public void init(ProcessorContext context) {
super.init(context);
store = (KeyValueStore<K, V>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<K, V>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<K, V>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues), sendOldValues);
}
/**
@@ -89,7 +89,7 @@ public class KTableReduce<K, V> implements KTableProcessorSupplier<K, V, V> {
// update the store with the new value
store.put(key, newAgg);
- tupleForwarder.maybeForward(key, newAgg, oldAgg, sendOldValues);
+ tupleForwarder.maybeForward(key, newAgg, oldAgg);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
index 7b777d1..d391151 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableSource.java
@@ -53,7 +53,7 @@ public class KTableSource<K, V> implements ProcessorSupplier<K, V> {
public void init(ProcessorContext context) {
super.init(context);
store = (KeyValueStore<K, V>) context.getStateStore(storeName);
- tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues));
+ tupleForwarder = new TupleForwarder<>(store, context, new ForwardingCacheFlushListener<K, V>(context, sendOldValues), sendOldValues);
}
@Override
@@ -63,7 +63,7 @@ public class KTableSource<K, V> implements ProcessorSupplier<K, V> {
throw new StreamsException("Record key for the source KTable from store name " + storeName + " should not be null.");
V oldValue = store.get(key);
store.put(key, value);
- tupleForwarder.maybeForward(key, value, oldValue, sendOldValues);
+ tupleForwarder.maybeForward(key, value, oldValue);
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionKeySerde.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionKeySerde.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionKeySerde.java
new file mode 100644
index 0000000..165d5c6
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionKeySerde.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serde;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.kstream.Windowed;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+/**
+ * Serde for a {@link Windowed} key when working with {@link org.apache.kafka.streams.kstream.SessionWindows}
+ *
+ * @param <K> sessionId type
+ */
+public class SessionKeySerde<K> implements Serde<Windowed<K>> {
+ private static final int TIMESTAMP_SIZE = 8;
+ private static final String SESSIONKEY = "sessionkey";
+
+ private final Serde<K> keySerde;
+
+ public SessionKeySerde(final Serde<K> keySerde) {
+ this.keySerde = keySerde;
+ }
+
+ @Override
+ public void configure(final Map<String, ?> configs, final boolean isKey) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Serializer<Windowed<K>> serializer() {
+ return new SessionKeySerializer(keySerde.serializer());
+ }
+
+ @Override
+ public Deserializer<Windowed<K>> deserializer() {
+ return new SessionKeyDeserializer(keySerde.deserializer());
+ }
+
+ private class SessionKeySerializer implements Serializer<Windowed<K>> {
+
+ private final Serializer<K> keySerializer;
+
+ SessionKeySerializer(final Serializer<K> keySerializer) {
+ this.keySerializer = keySerializer;
+ }
+
+ @Override
+ public void configure(final Map<String, ?> configs, final boolean isKey) {
+
+ }
+
+ @Override
+ public byte[] serialize(final String topic, final Windowed<K> data) {
+ if (data == null) {
+ return null;
+ }
+ return toBinary(data, keySerializer).get();
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+
+ private class SessionKeyDeserializer implements Deserializer<Windowed<K>> {
+ private final Deserializer<K> deserializer;
+
+ SessionKeyDeserializer(final Deserializer<K> deserializer) {
+ this.deserializer = deserializer;
+ }
+
+ @Override
+ public void configure(final Map<String, ?> configs, final boolean isKey) {
+ }
+
+ @Override
+ public Windowed<K> deserialize(final String topic, final byte[] data) {
+ if (data == null || data.length == 0) {
+ return null;
+ }
+ return from(data, deserializer);
+ }
+
+
+ @Override
+ public void close() {
+
+ }
+ }
+
+
+ public static long extractEnd(final byte [] binaryKey) {
+ return ByteBuffer.wrap(binaryKey).getLong(binaryKey.length - 2 * TIMESTAMP_SIZE);
+ }
+
+ public static long extractStart(final byte [] binaryKey) {
+ return ByteBuffer.wrap(binaryKey).getLong(binaryKey.length - TIMESTAMP_SIZE);
+ }
+
+ public static byte[] extractKeyBytes(final byte[] binaryKey) {
+ final byte[] bytes = new byte[binaryKey.length - 2 * TIMESTAMP_SIZE];
+ System.arraycopy(binaryKey, 0, bytes, 0, bytes.length);
+ return bytes;
+ }
+
+ public static <K> Windowed<K> from(final byte[] binaryKey, final Deserializer<K> keyDeserializer) {
+ final K key = extractKey(binaryKey, keyDeserializer);
+ final ByteBuffer buffer = ByteBuffer.wrap(binaryKey);
+ final long start = buffer.getLong(binaryKey.length - TIMESTAMP_SIZE);
+ final long end = buffer.getLong(binaryKey.length - 2 * TIMESTAMP_SIZE);
+ return new Windowed<>(key, new TimeWindow(start, end));
+ }
+
+ private static <K> K extractKey(final byte[] binaryKey, Deserializer<K> deserializer) {
+ return deserializer.deserialize(SESSIONKEY, extractKeyBytes(binaryKey));
+ }
+
+ public static <K> Bytes toBinary(final Windowed<K> sessionKey, final Serializer<K> serializer) {
+ final byte[] bytes = serializer.serialize(SESSIONKEY, sessionKey.key());
+ ByteBuffer buf = ByteBuffer.allocate(bytes.length + 2 * TIMESTAMP_SIZE);
+ buf.put(bytes);
+ buf.putLong(sessionKey.window().end());
+ buf.putLong(sessionKey.window().start());
+ return new Bytes(buf.array());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java
index 02609d7..4834acf 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TupleForwarder.java
@@ -21,16 +21,26 @@ import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.StateStore;
import org.apache.kafka.streams.state.internals.CachedStateStore;
-
+/**
+ * This class is used to determine if a processor should forward values to child nodes.
+ * Forwarding only occurs when caching is not enabled.
+ *
+ * @param <K>
+ * @param <V>
+ */
class TupleForwarder<K, V> {
private final boolean cached;
private final ProcessorContext context;
+ private final boolean sendOldValues;
+
@SuppressWarnings("unchecked")
- public TupleForwarder(final StateStore store,
- final ProcessorContext context,
- final ForwardingCacheFlushListener flushListener) {
+ TupleForwarder(final StateStore store,
+ final ProcessorContext context,
+ final ForwardingCacheFlushListener flushListener,
+ final boolean sendOldValues) {
this.cached = store instanceof CachedStateStore;
this.context = context;
+ this.sendOldValues = sendOldValues;
if (this.cached) {
((CachedStateStore) store).setFlushListener(flushListener);
}
@@ -38,8 +48,7 @@ class TupleForwarder<K, V> {
public void maybeForward(final K key,
final V newValue,
- final V oldValue,
- final boolean sendOldValues) {
+ final V oldValue) {
if (!cached) {
if (sendOldValues) {
context.forward(key, new Change<>(newValue, oldValue));
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
index 74fea9c..ca6b7b7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/TopologyBuilder.java
@@ -29,7 +29,7 @@ import org.apache.kafka.streams.processor.internals.QuickUnion;
import org.apache.kafka.streams.processor.internals.SinkNode;
import org.apache.kafka.streams.processor.internals.SourceNode;
import org.apache.kafka.streams.processor.internals.StreamPartitionAssignor.SubscriptionUpdates;
-import org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier;
+import org.apache.kafka.streams.state.internals.WindowStoreSupplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -912,11 +912,11 @@ public class TopologyBuilder {
}
private InternalTopicConfig createInternalTopicConfig(final StateStoreSupplier supplier, final String name) {
- if (!(supplier instanceof RocksDBWindowStoreSupplier)) {
+ if (!(supplier instanceof WindowStoreSupplier)) {
return new InternalTopicConfig(name, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), supplier.logConfig());
}
- final RocksDBWindowStoreSupplier windowStoreSupplier = (RocksDBWindowStoreSupplier) supplier;
+ final WindowStoreSupplier windowStoreSupplier = (WindowStoreSupplier) supplier;
final InternalTopicConfig config = new InternalTopicConfig(name,
Utils.mkSet(InternalTopicConfig.CleanupPolicy.compact,
InternalTopicConfig.CleanupPolicy.delete),
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
index ddbc7b3..f9bab2d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/KeyValueIterator.java
@@ -37,4 +37,10 @@ public interface KeyValueIterator<K, V> extends Iterator<KeyValue<K, V>>, Closea
@Override
void close();
+
+ /**
+ * Peek at the next key without advancing the iterator
+ * @return the key of the next value that would be returned from the next call to next
+ */
+ K peekNextKey();
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/QueryableStoreTypes.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/QueryableStoreTypes.java b/streams/src/main/java/org/apache/kafka/streams/state/QueryableStoreTypes.java
index d57fe35..a783e07 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/QueryableStoreTypes.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/QueryableStoreTypes.java
@@ -16,6 +16,7 @@ package org.apache.kafka.streams.state;
import org.apache.kafka.streams.processor.StateStore;
import org.apache.kafka.streams.state.internals.CompositeReadOnlyKeyValueStore;
+import org.apache.kafka.streams.state.internals.CompositeReadOnlySessionStore;
import org.apache.kafka.streams.state.internals.CompositeReadOnlyWindowStore;
import org.apache.kafka.streams.state.internals.StateStoreProvider;
@@ -46,6 +47,16 @@ public class QueryableStoreTypes {
return new WindowStoreType<>();
}
+ /**
+ * A {@link QueryableStoreType} that accepts {@link ReadOnlySessionStore}
+ * @param <K> key type of the store
+ * @param <V> value type of the store
+ * @return {@link SessionStoreType}
+ */
+ public static <K, V> QueryableStoreType<ReadOnlySessionStore<K, V>> sessionStore() {
+ return new SessionStoreType<>();
+ }
+
private static abstract class QueryableStoreTypeMatcher<T> implements QueryableStoreType<T> {
private final Class matchTo;
@@ -85,6 +96,15 @@ public class QueryableStoreTypes {
final String storeName) {
return new CompositeReadOnlyWindowStore<>(storeProvider, this, storeName);
}
+ }
+ private static class SessionStoreType<K, V> extends QueryableStoreTypeMatcher<ReadOnlySessionStore<K, V>> {
+ SessionStoreType() {
+ super(ReadOnlySessionStore.class);
+ }
+ @Override
+ public ReadOnlySessionStore<K, V> create(final StateStoreProvider storeProvider, final String storeName) {
+ return new CompositeReadOnlySessionStore<>(storeProvider, this, storeName);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java
new file mode 100644
index 0000000..f09d357
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlySessionStore.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state;
+
+
+import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.streams.kstream.Windowed;
+
+/**
+ * A session store that only supports read operations.
+ * Implementations should be thread-safe as concurrent reads and writes
+ * are expected.
+ *
+ * @param <K> the key type
+ * @param <AGG> the aggregated value type
+ */
+@InterfaceStability.Unstable
+public interface ReadOnlySessionStore<K, AGG> {
+
+ /**
+ * Retrieve all aggregated sessions for the provided key
+ * @param key record key to find aggregated session values for
+ * @return KeyValueIterator containing all sessions for the provided key.
+ */
+ KeyValueIterator<Windowed<K>, AGG> fetch(final K key);
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java
new file mode 100644
index 0000000..39658a3
--- /dev/null
+++ b/streams/src/main/java/org/apache/kafka/streams/state/SessionStore.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.state;
+
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.processor.StateStore;
+
+/**
+ * Interface for storing the aggregated values of sessions
+ * @param <K> type of the record keys
+ * @param <AGG> type of the aggregated values
+ */
+public interface SessionStore<K, AGG> extends StateStore, ReadOnlySessionStore<K, AGG> {
+
+ /**
+ * Fetch any sessions with the matching key and the sessions end is &le earliestEndTime and the sessions
+ * start is &ge latestStartTime
+ */
+ KeyValueIterator<Windowed<K>, AGG> findSessionsToMerge(final K key, long earliestSessionEndTime, final long latestSessionStartTime);
+
+ /**
+ * Remove the session aggregated with provided {@link Windowed} key from the store
+ * @param sessionKey key of the session to remove
+ */
+ void remove(final Windowed<K> sessionKey);
+
+ /**
+ * Write the aggregated value for the provided key to the store
+ * @param sessionKey key of the session to write
+ * @param aggregate the aggregated value for the session
+ */
+ void put(final Windowed<K> sessionKey, final AGG aggregate);
+}
[2/5] kafka git commit: KAFKA-3452: Support session windows
Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
new file mode 100644
index 0000000..c107c3e
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSessionWindowAggregateProcessorTest.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.Aggregator;
+import org.apache.kafka.streams.kstream.Initializer;
+import org.apache.kafka.streams.kstream.KStreamBuilder;
+import org.apache.kafka.streams.kstream.Merger;
+import org.apache.kafka.streams.kstream.SessionWindows;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.processor.Processor;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.internals.RocksDBSessionStoreSupplier;
+import org.apache.kafka.streams.state.SessionStore;
+import org.apache.kafka.streams.state.internals.ThreadCache;
+import org.apache.kafka.test.KStreamTestDriver;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@SuppressWarnings("unchecked")
+public class KStreamSessionWindowAggregateProcessorTest {
+
+ private static final long GAP_MS = 5 * 60 * 1000L;
+ private static final String STORE_NAME = "session-store";
+
+ private final Initializer<Long> initializer = new Initializer<Long>() {
+ @Override
+ public Long apply() {
+ return 0L;
+ }
+ };
+ private final Aggregator<String, String, Long> aggregator = new Aggregator<String, String, Long>() {
+ @Override
+ public Long apply(final String aggKey, final String value, final Long aggregate) {
+ return aggregate + 1;
+ }
+ };
+ private final Merger<String, Long> sessionMerger = new Merger<String, Long>() {
+ @Override
+ public Long apply(final String aggKey, final Long aggOne, final Long aggTwo) {
+ return aggOne + aggTwo;
+ }
+ };
+ private final KStreamSessionWindowAggregate<String, String, Long> sessionAggregator =
+ new KStreamSessionWindowAggregate<>(SessionWindows.with(GAP_MS).until(3 * GAP_MS),
+ STORE_NAME,
+ initializer,
+ aggregator,
+ sessionMerger);
+
+ private final List<KeyValue> results = new ArrayList<>();
+ private Processor<String, String> processor = sessionAggregator.get();
+ private SessionStore<String, Long> sessionStore;
+ private MockProcessorContext context;
+
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void initializeStore() {
+ final File stateDir = TestUtils.tempDirectory();
+ context = new MockProcessorContext(new KStreamTestDriver(new KStreamBuilder(), stateDir), stateDir, Serdes.String(), Serdes.String(), new NoOpRecordCollector(), new ThreadCache(100000)) {
+ @Override
+ public <K, V> void forward(final K key, final V value) {
+ results.add(KeyValue.pair(key, value));
+ }
+ };
+
+ initStore(true);
+ processor.init(context);
+ }
+
+ private void initStore(final boolean enableCaching) {
+ final RocksDBSessionStoreSupplier<String, Long> supplier =
+ new RocksDBSessionStoreSupplier<>(STORE_NAME,
+ GAP_MS * 3,
+ Serdes.String(),
+ Serdes.Long(),
+ false,
+ Collections.<String, String>emptyMap(),
+ enableCaching);
+ sessionStore = (SessionStore<String, Long>) supplier.get();
+ sessionStore.init(context, sessionStore);
+ }
+
+ @After
+ public void closeStore() {
+ sessionStore.close();
+ }
+
+ @Test
+ public void shouldCreateSingleSessionWhenWithinGap() throws Exception {
+ context.setTime(0);
+ processor.process("john", "first");
+ context.setTime(500);
+ processor.process("john", "second");
+
+ final KeyValueIterator<Windowed<String>, Long> values = sessionStore.findSessionsToMerge("john", 0, 2000);
+ assertTrue(values.hasNext());
+ assertEquals(Long.valueOf(2), values.next().value);
+ }
+
+
+ @Test
+ public void shouldMergeSessions() throws Exception {
+ context.setTime(0);
+ final String sessionId = "mel";
+ processor.process(sessionId, "first");
+ assertTrue(sessionStore.findSessionsToMerge(sessionId, 0, 0).hasNext());
+
+ // move time beyond gap
+ context.setTime(GAP_MS + 1);
+ processor.process(sessionId, "second");
+ assertTrue(sessionStore.findSessionsToMerge(sessionId, GAP_MS + 1, GAP_MS + 1).hasNext());
+ // should still exist as not within gap
+ assertTrue(sessionStore.findSessionsToMerge(sessionId, 0, 0).hasNext());
+ // move time back
+ context.setTime(GAP_MS / 2);
+ processor.process(sessionId, "third");
+
+ final KeyValueIterator<Windowed<String>, Long> iterator = sessionStore.findSessionsToMerge(sessionId, 0, GAP_MS + 1);
+ final KeyValue<Windowed<String>, Long> kv = iterator.next();
+
+ assertEquals(Long.valueOf(3), kv.value);
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test
+ public void shouldUpdateSessionIfTheSameTime() throws Exception {
+ context.setTime(0);
+ processor.process("mel", "first");
+ processor.process("mel", "second");
+ final KeyValueIterator<Windowed<String>, Long> iterator = sessionStore.findSessionsToMerge("mel", 0, 0);
+ assertEquals(Long.valueOf(2L), iterator.next().value);
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test
+ public void shouldHaveMultipleSessionsForSameIdWhenTimestampApartBySessionGap() throws Exception {
+ final String sessionId = "mel";
+ long time = 0;
+ context.setTime(time);
+ processor.process(sessionId, "first");
+ context.setTime(time += GAP_MS + 1);
+ processor.process(sessionId, "second");
+ processor.process(sessionId, "second");
+ context.setTime(time += GAP_MS + 1);
+ processor.process(sessionId, "third");
+ processor.process(sessionId, "third");
+ processor.process(sessionId, "third");
+
+ sessionStore.flush();
+ assertEquals(Arrays.asList(
+ KeyValue.pair(new Windowed<>(sessionId, new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>(sessionId, new TimeWindow(GAP_MS + 1, GAP_MS + 1)), new Change<>(2L, null)),
+ KeyValue.pair(new Windowed<>(sessionId, new TimeWindow(time, time)), new Change<>(3L, null))
+
+ ), results);
+
+ }
+
+
+ @Test
+ public void shouldRemoveMergedSessionsFromStateStore() throws Exception {
+ context.setTime(0);
+ processor.process("a", "1");
+
+ // first ensure it is in the store
+ final KeyValueIterator<Windowed<String>, Long> a1 = sessionStore.findSessionsToMerge("a", 0, 0);
+ assertEquals(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), 1L), a1.next());
+
+ context.setTime(100);
+ processor.process("a", "2");
+ // a1 from above should have been removed
+ // should have merged session in store
+ final KeyValueIterator<Windowed<String>, Long> a2 = sessionStore.findSessionsToMerge("a", 0, 100);
+ assertEquals(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 100)), 2L), a2.next());
+ assertFalse(a2.hasNext());
+ }
+
+ @Test
+ public void shouldHandleMultipleSessionsAndMerging() throws Exception {
+ context.setTime(0);
+ processor.process("a", "1");
+ processor.process("b", "1");
+ processor.process("c", "1");
+ processor.process("d", "1");
+ context.setTime(GAP_MS / 2);
+ processor.process("d", "2");
+ context.setTime(GAP_MS + 1);
+ processor.process("a", "2");
+ processor.process("b", "2");
+ context.setTime(GAP_MS + 1 + GAP_MS / 2);
+ processor.process("a", "3");
+ processor.process("c", "3");
+
+ sessionStore.flush();
+
+ assertEquals(Arrays.asList(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("b", new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("c", new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("d", new TimeWindow(0, GAP_MS / 2)), new Change<>(2L, null)),
+ KeyValue.pair(new Windowed<>("b", new TimeWindow(GAP_MS + 1, GAP_MS + 1)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(GAP_MS + 1, GAP_MS + 1 + GAP_MS / 2)), new Change<>(2L, null)),
+ KeyValue.pair(new Windowed<>("c", new TimeWindow(GAP_MS + 1 + GAP_MS / 2, GAP_MS + 1 + GAP_MS / 2)), new Change<>(1L, null))
+ ),
+ results);
+ }
+
+
+ @Test
+ public void shouldGetAggregatedValuesFromValueGetter() throws Exception {
+ final KTableValueGetter<Windowed<String>, Long> getter = sessionAggregator.view().get();
+ getter.init(context);
+ context.setTime(0);
+ processor.process("a", "1");
+ context.setTime(GAP_MS + 1);
+ processor.process("a", "1");
+ processor.process("a", "2");
+ final long t0 = getter.get(new Windowed<>("a", new TimeWindow(0, 0)));
+ final long t1 = getter.get(new Windowed<>("a", new TimeWindow(GAP_MS + 1, GAP_MS + 1)));
+ assertEquals(1L, t0);
+ assertEquals(2L, t1);
+ }
+
+ @Test
+ public void shouldImmediatelyForwardNewSessionWhenNonCachedStore() throws Exception {
+ initStore(false);
+ processor.init(context);
+
+ context.setTime(0);
+ processor.process("a", "1");
+ processor.process("b", "1");
+ processor.process("c", "1");
+
+ assertEquals(Arrays.asList(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("b", new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("c", new TimeWindow(0, 0)), new Change<>(1L, null))), results);
+ }
+
+ @Test
+ public void shouldImmediatelyForwardRemovedSessionsWhenMerging() throws Exception {
+ initStore(false);
+ processor.init(context);
+
+ context.setTime(0);
+ processor.process("a", "1");
+ context.setTime(5);
+ processor.process("a", "1");
+ assertEquals(Arrays.asList(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), new Change<>(1L, null)),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), new Change<>(null, null)),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 5)), new Change<>(2L, null))), results);
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionKeySerdeTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionKeySerdeTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionKeySerdeTest.java
new file mode 100644
index 0000000..2f5972c
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionKeySerdeTest.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.kstream.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class SessionKeySerdeTest {
+
+ @Test
+ public void shouldSerializeDeserialize() throws Exception {
+ final Windowed<Long> key = new Windowed<>(1L, new TimeWindow(10, 100));
+ final SessionKeySerde<Long> serde = new SessionKeySerde<>(Serdes.Long());
+ final byte[] bytes = serde.serializer().serialize("t", key);
+ final Windowed<Long> result = serde.deserializer().deserialize("t", bytes);
+ assertEquals(key, result);
+ }
+
+ @Test
+ public void shouldSerializeNullToNull() throws Exception {
+ final SessionKeySerde<String> serde = new SessionKeySerde<>(Serdes.String());
+ assertNull(serde.serializer().serialize("t", null));
+ }
+
+ @Test
+ public void shouldDeSerializeEmtpyByteArrayToNull() throws Exception {
+ final SessionKeySerde<String> serde = new SessionKeySerde<>(Serdes.String());
+ assertNull(serde.deserializer().deserialize("t", new byte[0]));
+ }
+
+ @Test
+ public void shouldDeSerializeNullToNull() throws Exception {
+ final SessionKeySerde<String> serde = new SessionKeySerde<>(Serdes.String());
+ assertNull(serde.deserializer().deserialize("t", null));
+ }
+
+ @Test
+ public void shouldConvertToBinaryAndBack() throws Exception {
+ final Windowed<String> key = new Windowed<>("key", new TimeWindow(10, 20));
+ final Bytes serialized = SessionKeySerde.toBinary(key, Serdes.String().serializer());
+ final Windowed<String> result = SessionKeySerde.from(serialized.get(), Serdes.String().deserializer());
+ assertEquals(key, result);
+ }
+
+ @Test
+ public void shouldExtractEndTimeFromBinary() throws Exception {
+ final Windowed<String> key = new Windowed<>("key", new TimeWindow(10, 100));
+ final Bytes serialized = SessionKeySerde.toBinary(key, Serdes.String().serializer());
+ assertEquals(100, SessionKeySerde.extractEnd(serialized.get()));
+ }
+
+ @Test
+ public void shouldExtractStartTimeFromBinary() throws Exception {
+ final Windowed<String> key = new Windowed<>("key", new TimeWindow(50, 100));
+ final Bytes serialized = SessionKeySerde.toBinary(key, Serdes.String().serializer());
+ assertEquals(50, SessionKeySerde.extractStart(serialized.get()));
+ }
+
+ @Test
+ public void shouldExtractKeyBytesFromBinary() throws Exception {
+ final Windowed<String> key = new Windowed<>("blah", new TimeWindow(50, 100));
+ final Bytes serialized = SessionKeySerde.toBinary(key, Serdes.String().serializer());
+ assertArrayEquals("blah".getBytes(), SessionKeySerde.extractKeyBytes(serialized.get()));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java
new file mode 100644
index 0000000..cb6f87e
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingSessionStoreTest.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.SessionKeySerde;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
+import org.apache.kafka.streams.processor.internals.RecordCollector;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.StateSerdes;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.kafka.streams.state.internals.RocksDBSessionStoreTest.toList;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+
+public class CachingSessionStoreTest {
+
+ private static final int MAX_CACHE_SIZE_BYTES = 600;
+ private final StateSerdes<String, Long> serdes =
+ new StateSerdes<>("name", Serdes.String(), Serdes.Long());
+ private RocksDBSegmentedBytesStore underlying;
+ private CachingSessionStore<String, Long> cachingStore;
+ private ThreadCache cache;
+ private static final Long DEFAULT_TIMESTAMP = 10L;
+
+ @Before
+ public void setUp() throws Exception {
+ underlying = new RocksDBSegmentedBytesStore("test", 60000, 3, new SessionKeySchema());
+ cachingStore = new CachingSessionStore<>(underlying,
+ Serdes.String(),
+ Serdes.Long());
+ cache = new ThreadCache(MAX_CACHE_SIZE_BYTES);
+ final MockProcessorContext context = new MockProcessorContext(null, TestUtils.tempDirectory(), null, null, (RecordCollector) null, cache);
+ context.setRecordContext(new ProcessorRecordContext(DEFAULT_TIMESTAMP, 0, 0, "topic"));
+ cachingStore.init(context, cachingStore);
+ }
+
+ @Test
+ public void shouldPutFetchFromCache() throws Exception {
+ cachingStore.put(new Windowed<>("a", new TimeWindow(0, 0)), 1L);
+ cachingStore.put(new Windowed<>("aa", new TimeWindow(0, 0)), 1L);
+ cachingStore.put(new Windowed<>("b", new TimeWindow(0, 0)), 1L);
+
+ final KeyValueIterator<Windowed<String>, Long> a = cachingStore.findSessionsToMerge("a", 0, 0);
+ final KeyValueIterator<Windowed<String>, Long> b = cachingStore.findSessionsToMerge("b", 0, 0);
+
+ assertEquals(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), 1L), a.next());
+ assertEquals(KeyValue.pair(new Windowed<>("b", new TimeWindow(0, 0)), 1L), b.next());
+ assertFalse(a.hasNext());
+ assertFalse(b.hasNext());
+ assertEquals(3, cache.size());
+ }
+
+ @Test
+ public void shouldFetchAllSessionsWithSameRecordKey() throws Exception {
+
+ final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), 1L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(10, 10)), 2L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(100, 100)), 3L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(1000, 1000)), 4L));
+ for (KeyValue<Windowed<String>, Long> kv : expected) {
+ cachingStore.put(kv.key, kv.value);
+ }
+
+ // add one that shouldn't appear in the results
+ cachingStore.put(new Windowed<>("aa", new TimeWindow(0, 0)), 5L);
+
+
+ final List<KeyValue<Windowed<String>, Long>> results = toList(cachingStore.fetch("a"));
+ assertEquals(expected, results);
+
+ }
+
+ @Test
+ public void shouldFlushItemsToStoreOnEviction() throws Exception {
+ final List<KeyValue<Windowed<String>, Long>> added = addSessionsUntilOverflow("a", "b", "c", "d");
+ assertEquals(added.size() - 1, cache.size());
+ final KeyValueIterator<Bytes, byte[]> iterator = underlying.fetch(Bytes.wrap(added.get(0).key.key().getBytes()), 0, 0);
+ final KeyValue<Bytes, byte[]> next = iterator.next();
+ assertEquals(added.get(0).key, SessionKeySerde.from(next.key.get(), Serdes.String().deserializer()));
+ assertArrayEquals(serdes.rawValue(added.get(0).value), next.value);
+ }
+
+ @Test
+ public void shouldQueryItemsInCacheAndStore() throws Exception {
+ final List<KeyValue<Windowed<String>, Long>> added = addSessionsUntilOverflow("a");
+ final KeyValueIterator<Windowed<String>, Long> iterator = cachingStore.findSessionsToMerge("a", 0, added.size() * 10);
+ final List<KeyValue<Windowed<String>, Long>> actual = toList(iterator);
+ assertEquals(added, actual);
+ }
+
+ @Test
+ public void shouldRemove() throws Exception {
+ final Windowed<String> a = new Windowed<>("a", new TimeWindow(0, 0));
+ final Windowed<String> b = new Windowed<>("b", new TimeWindow(0, 0));
+ cachingStore.put(a, 2L);
+ cachingStore.put(b, 2L);
+ cachingStore.flush();
+ cachingStore.remove(a);
+ cachingStore.flush();
+ final KeyValueIterator<Windowed<String>, Long> rangeIter = cachingStore.findSessionsToMerge("a", 0, 0);
+ assertFalse(rangeIter.hasNext());
+ }
+
+ @Test
+ public void shouldFetchCorrectlyAcrossSegments() throws Exception {
+ final Windowed<String> a1 = new Windowed<>("a", new TimeWindow(0, 0));
+ final Windowed<String> a2 = new Windowed<>("a", new TimeWindow(Segments.MIN_SEGMENT_INTERVAL, Segments.MIN_SEGMENT_INTERVAL));
+ final Windowed<String> a3 = new Windowed<>("a", new TimeWindow(Segments.MIN_SEGMENT_INTERVAL * 2, Segments.MIN_SEGMENT_INTERVAL * 2));
+ cachingStore.put(a1, 1L);
+ cachingStore.put(a2, 2L);
+ cachingStore.put(a3, 3L);
+ cachingStore.flush();
+ final KeyValueIterator<Windowed<String>, Long> results = cachingStore.findSessionsToMerge("a", 0, Segments.MIN_SEGMENT_INTERVAL * 2);
+ assertEquals(a1, results.next().key);
+ assertEquals(a2, results.next().key);
+ assertEquals(a3, results.next().key);
+ assertFalse(results.hasNext());
+ }
+
+ @Test
+ public void shouldClearNamespaceCacheOnClose() throws Exception {
+ final Windowed<String> a1 = new Windowed<>("a", new TimeWindow(0, 0));
+ cachingStore.put(a1, 1L);
+ assertEquals(1, cache.size());
+ cachingStore.close();
+ assertEquals(0, cache.size());
+ }
+
+ @Test(expected = InvalidStateStoreException.class)
+ public void shouldThrowIfTryingToFetchFromClosedCachingStore() throws Exception {
+ cachingStore.close();
+ cachingStore.fetch("a");
+ }
+
+ @Test(expected = InvalidStateStoreException.class)
+ public void shouldThrowIfTryingToFindMergeSessionFromClosedCachingStore() throws Exception {
+ cachingStore.close();
+ cachingStore.findSessionsToMerge("a", 0, Long.MAX_VALUE);
+ }
+
+ @Test(expected = InvalidStateStoreException.class)
+ public void shouldThrowIfTryingToRemoveFromClosedCachingStore() throws Exception {
+ cachingStore.close();
+ cachingStore.remove(new Windowed<>("a", new TimeWindow(0, 0)));
+ }
+
+ @Test(expected = InvalidStateStoreException.class)
+ public void shouldThrowIfTryingToPutIntoClosedCachingStore() throws Exception {
+ cachingStore.close();
+ cachingStore.put(new Windowed<>("a", new TimeWindow(0, 0)), 1L);
+ }
+
+ private List<KeyValue<Windowed<String>, Long>> addSessionsUntilOverflow(final String...sessionIds) {
+ final Random random = new Random();
+ final List<KeyValue<Windowed<String>, Long>> results = new ArrayList<>();
+ while (cache.size() == results.size()) {
+ final String sessionId = sessionIds[random.nextInt(sessionIds.length)];
+ addSingleSession(sessionId, results);
+ }
+ return results;
+ }
+
+ private void addSingleSession(final String sessionId, final List<KeyValue<Windowed<String>, Long>> allSessions) {
+ final int timestamp = allSessions.size() * 10;
+ final Windowed<String> key = new Windowed<>(sessionId, new TimeWindow(timestamp, timestamp));
+ final Long value = 1L;
+ cachingStore.put(key, value);
+ allSessions.add(KeyValue.pair(key, value));
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
index 023fea6..427798d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingWindowStoreTest.java
@@ -25,6 +25,7 @@ import org.apache.kafka.streams.kstream.Windowed;
import org.apache.kafka.streams.kstream.internals.TimeWindow;
import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
import org.apache.kafka.streams.processor.internals.RecordCollector;
+import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.WindowStoreIterator;
import org.apache.kafka.test.MockProcessorContext;
import org.apache.kafka.test.TestUtils;
@@ -44,16 +45,18 @@ public class CachingWindowStoreTest {
private static final int MAX_CACHE_SIZE_BYTES = 150;
private static final Long WINDOW_SIZE = 10000L;
- private RocksDBWindowStore<Bytes, byte[]> underlying;
+ private RocksDBSegmentedBytesStore underlying;
private CachingWindowStore<String, String> cachingStore;
private CachingKeyValueStoreTest.CacheFlushListenerStub<Windowed<String>> cacheListener;
private ThreadCache cache;
private String topic;
- private static final Long DEFAULT_TIMESTAMP = 10L;
+ private static final long DEFAULT_TIMESTAMP = 10L;
+ private WindowStoreKeySchema keySchema;
@Before
public void setUp() throws Exception {
- underlying = new RocksDBWindowStore<>("test", 30000, 3, false, Serdes.Bytes(), Serdes.ByteArray());
+ keySchema = new WindowStoreKeySchema();
+ underlying = new RocksDBSegmentedBytesStore("test", 30000, 3, keySchema);
cacheListener = new CachingKeyValueStoreTest.CacheFlushListenerStub<>();
cachingStore = new CachingWindowStore<>(underlying,
Serdes.String(),
@@ -85,9 +88,9 @@ public class CachingWindowStoreTest {
public void shouldFlushEvictedItemsIntoUnderlyingStore() throws Exception {
int added = addItemsToCache();
// all dirty entries should have been flushed
- final WindowStoreIterator<byte[]> iter = underlying.fetch(Bytes.wrap("0".getBytes()), DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP);
- final KeyValue<Long, byte[]> next = iter.next();
- assertEquals(DEFAULT_TIMESTAMP, next.key);
+ final KeyValueIterator<Bytes, byte[]> iter = underlying.fetch(Bytes.wrap("0".getBytes()), DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP);
+ final KeyValue<Bytes, byte[]> next = iter.next();
+ assertEquals(DEFAULT_TIMESTAMP, keySchema.segmentTimestamp(next.key));
assertArrayEquals("0".getBytes(), next.value);
assertFalse(iter.hasNext());
assertEquals(added - 1, cache.size());
@@ -143,7 +146,8 @@ public class CachingWindowStoreTest {
@Test
public void shouldIterateCacheAndStore() throws Exception {
- underlying.put(Bytes.wrap("1".getBytes()), "a".getBytes());
+ final Bytes key = Bytes.wrap("1" .getBytes());
+ underlying.put(Bytes.wrap(WindowStoreUtils.toBinaryKey(key, DEFAULT_TIMESTAMP, 0, WindowStoreUtils.INNER_SERDES)), "a".getBytes());
cachingStore.put("1", "b", DEFAULT_TIMESTAMP + WINDOW_SIZE);
final WindowStoreIterator<String> fetch = cachingStore.fetch("1", DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP + WINDOW_SIZE);
assertEquals(KeyValue.pair(DEFAULT_TIMESTAMP, "a"), fetch.next());
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStoreTest.java
new file mode 100644
index 0000000..d4f9e47
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/ChangeLoggingSegmentedBytesStoreTest.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.SegmentedBytesStoreStub;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class ChangeLoggingSegmentedBytesStoreTest {
+
+ private final SegmentedBytesStoreStub bytesStore = new SegmentedBytesStoreStub();
+ private final ChangeLoggingSegmentedBytesStore store = new ChangeLoggingSegmentedBytesStore(bytesStore);
+ private final Map sent = new HashMap<>();
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setUp() throws Exception {
+ final NoOpRecordCollector collector = new NoOpRecordCollector() {
+ @Override
+ public <K, V> void send(final ProducerRecord<K, V> record, final Serializer<K> keySerializer, final Serializer<V> valueSerializer) {
+ sent.put(record.key(), record.value());
+ }
+ };
+ final MockProcessorContext context = new MockProcessorContext(null,
+ TestUtils.tempDirectory(),
+ Serdes.String(),
+ Serdes.Long(),
+ collector,
+ new ThreadCache(0));
+ context.setTime(0);
+ store.init(context, store);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void shouldLogPuts() throws Exception {
+ final byte[] value1 = {0};
+ final byte[] value2 = {1};
+ final Bytes key1 = Bytes.wrap(value1);
+ final Bytes key2 = Bytes.wrap(value2);
+ store.put(key1, value1);
+ store.put(key2, value2);
+ store.flush();
+ assertArrayEquals(value1, (byte[]) sent.get(key1));
+ assertArrayEquals(value2, (byte[]) sent.get(key2));
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void shouldLogRemoves() throws Exception {
+ final Bytes key1 = Bytes.wrap(new byte[]{0});
+ final Bytes key2 = Bytes.wrap(new byte[]{1});
+ store.remove(key1);
+ store.remove(key2);
+ store.flush();
+ assertTrue(sent.containsKey(key1));
+ assertTrue(sent.containsKey(key2));
+ assertNull(sent.get(key1));
+ assertNull(sent.get(key2));
+ }
+
+ @Test
+ public void shouldDelegateToUnderlyingStoreWhenFetching() throws Exception {
+ store.fetch(Bytes.wrap(new byte[0]), 1, 1);
+ assertTrue(bytesStore.fetchCalled);
+ }
+
+ @Test
+ public void shouldFlushUnderlyingStore() throws Exception {
+ store.flush();
+ assertTrue(bytesStore.flushed);
+ }
+
+ @Test
+ public void shouldCloseUnderlyingStore() throws Exception {
+ store.close();
+ assertTrue(bytesStore.closed);
+ }
+
+ @Test
+ public void shouldInitUnderlyingStore() throws Exception {
+ assertTrue(bytesStore.initialized);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStoreTest.java
new file mode 100644
index 0000000..fc4a4c5
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlySessionStoreTest.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.errors.InvalidStateStoreException;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.QueryableStoreTypes;
+import org.apache.kafka.test.ReadOnlySessionStoreStub;
+import org.apache.kafka.test.StateStoreProviderStub;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.kafka.streams.state.internals.CompositeReadOnlyWindowStoreTest.toList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class CompositeReadOnlySessionStoreTest {
+
+ private final String storeName = "session-store";
+ private final StateStoreProviderStub stubProviderOne = new StateStoreProviderStub(false);
+ private final StateStoreProviderStub stubProviderTwo = new StateStoreProviderStub(false);
+ private final ReadOnlySessionStoreStub<String, Long> underlyingSessionStore = new ReadOnlySessionStoreStub<>();
+ private final ReadOnlySessionStoreStub<String, Long> otherUnderlyingStore = new ReadOnlySessionStoreStub<>();
+ private CompositeReadOnlySessionStore<String, Long> sessionStore;
+
+ @Before
+ public void before() {
+ stubProviderOne.addStore(storeName, underlyingSessionStore);
+ stubProviderOne.addStore("other-session-store", otherUnderlyingStore);
+
+
+ sessionStore = new CompositeReadOnlySessionStore<>(
+ new WrappingStoreProvider(Arrays.<StateStoreProvider>asList(stubProviderOne, stubProviderTwo)),
+ QueryableStoreTypes.<String, Long>sessionStore(), storeName);
+ }
+
+ @Test
+ public void shouldFetchResulstFromUnderlyingSessionStore() throws Exception {
+ underlyingSessionStore.put(new Windowed<>("a", new TimeWindow(0, 0)), 1L);
+ underlyingSessionStore.put(new Windowed<>("a", new TimeWindow(10, 10)), 2L);
+
+ final List<KeyValue<Windowed<String>, Long>> results = toList(sessionStore.fetch("a"));
+ assertEquals(Arrays.asList(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), 1L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(10, 10)), 2L)),
+ results);
+ }
+
+ @Test
+ public void shouldReturnEmptyIteratorIfNoData() throws Exception {
+ final KeyValueIterator<Windowed<String>, Long> result = sessionStore.fetch("b");
+ assertFalse(result.hasNext());
+ }
+
+ @Test
+ public void shouldFindValueForKeyWhenMultiStores() throws Exception {
+ final ReadOnlySessionStoreStub<String, Long> secondUnderlying = new
+ ReadOnlySessionStoreStub<>();
+ stubProviderTwo.addStore(storeName, secondUnderlying);
+
+ final Windowed<String> keyOne = new Windowed<>("key-one", new TimeWindow(0, 0));
+ final Windowed<String> keyTwo = new Windowed<>("key-two", new TimeWindow(0, 0));
+ underlyingSessionStore.put(keyOne, 0L);
+ secondUnderlying.put(keyTwo, 10L);
+
+ final List<KeyValue<Windowed<String>, Long>> keyOneResults = toList(sessionStore.fetch("key-one"));
+ final List<KeyValue<Windowed<String>, Long>> keyTwoResults = toList(sessionStore.fetch("key-two"));
+
+ assertEquals(Collections.singletonList(KeyValue.pair(keyOne, 0L)), keyOneResults);
+ assertEquals(Collections.singletonList(KeyValue.pair(keyTwo, 10L)), keyTwoResults);
+ }
+
+ @Test
+ public void shouldNotGetValueFromOtherStores() throws Exception {
+ final Windowed<String> expectedKey = new Windowed<>("foo", new TimeWindow(0, 0));
+ otherUnderlyingStore.put(new Windowed<>("foo", new TimeWindow(10, 10)), 10L);
+ underlyingSessionStore.put(expectedKey, 1L);
+
+ final KeyValueIterator<Windowed<String>, Long> result = sessionStore.fetch("foo");
+ assertEquals(KeyValue.pair(expectedKey, 1L), result.next());
+ assertFalse(result.hasNext());
+ }
+
+ @Test(expected = InvalidStateStoreException.class)
+ public void shouldThrowInvalidStateStoreExceptionOnRebalance() throws Exception {
+ final CompositeReadOnlySessionStore<String, String> store
+ = new CompositeReadOnlySessionStore<>(new StateStoreProviderStub(true),
+ QueryableStoreTypes.<String, String>sessionStore(),
+ "whateva");
+
+ store.fetch("a");
+ }
+
+ @Test(expected = InvalidStateStoreException.class)
+ public void shouldThrowInvalidStateStoreExceptionIfFetchThrows() throws Exception {
+ underlyingSessionStore.setOpen(false);
+ underlyingSessionStore.fetch("key");
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIteratorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIteratorTest.java
index 4a32187..bca4837 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingKeyValueIteratorTest.java
@@ -27,17 +27,18 @@ import static org.junit.Assert.assertTrue;
public class DelegatingPeekingKeyValueIteratorTest {
+ private final String name = "name";
private InMemoryKeyValueStore<String, String> store;
@Before
public void setUp() throws Exception {
- store = new InMemoryKeyValueStore<>("name");
+ store = new InMemoryKeyValueStore<>(name);
}
@Test
public void shouldPeekNext() throws Exception {
store.put("A", "A");
- final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(store.all());
+ final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(name, store.all());
assertEquals("A", peekingIterator.peekNextKey());
assertEquals("A", peekingIterator.peekNextKey());
assertTrue(peekingIterator.hasNext());
@@ -50,7 +51,7 @@ public class DelegatingPeekingKeyValueIteratorTest {
store.put(kv, kv);
}
- final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(store.all());
+ final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(name, store.all());
int index = 0;
while (peekingIterator.hasNext()) {
final String peekNext = peekingIterator.peekNextKey();
@@ -64,13 +65,13 @@ public class DelegatingPeekingKeyValueIteratorTest {
@Test(expected = NoSuchElementException.class)
public void shouldThrowNoSuchElementWhenNoMoreItemsLeftAndNextCalled() throws Exception {
- final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(store.all());
+ final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(name, store.all());
peekingIterator.next();
}
@Test(expected = NoSuchElementException.class)
public void shouldThrowNoSuchElementWhenNoMoreItemsLeftAndPeekNextCalled() throws Exception {
- final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(store.all());
+ final DelegatingPeekingKeyValueIterator<String, String> peekingIterator = new DelegatingPeekingKeyValueIterator<>(name, store.all());
peekingIterator.peekNextKey();
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIteratorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIteratorTest.java
deleted file mode 100644
index 3f251b3..0000000
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/DelegatingPeekingWindowIteratorTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kafka.streams.state.internals;
-
-import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
-import org.apache.kafka.streams.processor.internals.RecordCollector;
-import org.apache.kafka.streams.state.WindowStore;
-import org.apache.kafka.test.MockProcessorContext;
-import org.apache.kafka.test.TestUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class DelegatingPeekingWindowIteratorTest {
-
- private static final long DEFAULT_TIMESTAMP = 0L;
- private WindowStore<String, String> store;
-
- @Before
- public void setUp() throws Exception {
- store = new RocksDBWindowStore<>("test", 30000, 3, false, Serdes.String(), Serdes.String());
- final MockProcessorContext context = new MockProcessorContext(null, TestUtils.tempDirectory(), null, null, (RecordCollector) null, null);
- context.setRecordContext(new ProcessorRecordContext(DEFAULT_TIMESTAMP, 0, 0, "topic"));
- store.init(context, store);
- }
-
- @Test
- public void shouldPeekNext() throws Exception {
- final KeyValue<Long, String> expected = KeyValue.pair(DEFAULT_TIMESTAMP, "A");
- store.put("A", "A");
- final DelegatingPeekingWindowIterator<String> peekingIterator = new DelegatingPeekingWindowIterator<>(store.fetch("A", DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP));
- assertEquals(expected, peekingIterator.peekNext());
- assertEquals(expected, peekingIterator.peekNext());
- assertTrue(peekingIterator.hasNext());
- }
-
- @Test
- public void shouldPeekAndIterate() throws Exception {
- final List<KeyValue<Long, String>> expected = new ArrayList<>();
- for (long t = 0; t < 50; t += 10) {
- store.put("a", String.valueOf(t), t);
- expected.add(KeyValue.pair(t, String.valueOf(t)));
- }
- final DelegatingPeekingWindowIterator<String> peekingIterator = new DelegatingPeekingWindowIterator<>(store.fetch("a", 0, 50));
- int index = 0;
- while (peekingIterator.hasNext()) {
- final KeyValue<Long, String> peekNext = peekingIterator.peekNext();
- final KeyValue<Long, String> key = peekingIterator.next();
- assertEquals(expected.get(index), peekNext);
- assertEquals(expected.get(index), key);
- index++;
- }
- assertEquals(expected.size(), index);
- }
-
- @Test(expected = NoSuchElementException.class)
- public void shouldThrowNoSuchElementWhenNoMoreItemsLeftAndNextCalled() throws Exception {
- final DelegatingPeekingWindowIterator<String> peekingIterator = new DelegatingPeekingWindowIterator<>(store.fetch("b", 10, 10));
- peekingIterator.next();
- }
-
- @Test(expected = NoSuchElementException.class)
- public void shouldThrowNoSuchElementWhenNoMoreItemsLeftAndPeekNextCalled() throws Exception {
- final DelegatingPeekingWindowIterator<String> peekingIterator = new DelegatingPeekingWindowIterator<>(store.fetch("b", 10, 10));
- peekingIterator.peekNext();
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
index c7a1a2c..6bb27b7 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStore.java
@@ -103,12 +103,12 @@ class InMemoryKeyValueStore<K, V> implements KeyValueStore<K, V> {
@Override
public KeyValueIterator<K, V> range(final K from, final K to) {
- return new TheIterator(this.map.subMap(from, true, to, false).entrySet().iterator());
+ return new DelegatingPeekingKeyValueIterator<>(name, new TheIterator(this.map.subMap(from, true, to, false).entrySet().iterator()));
}
@Override
public KeyValueIterator<K, V> all() {
- return new TheIterator(map.entrySet().iterator());
+ return new DelegatingPeekingKeyValueIterator<>(name, new TheIterator(map.entrySet().iterator()));
}
private class TheIterator implements KeyValueIterator<K, V> {
@@ -125,6 +125,11 @@ class InMemoryKeyValueStore<K, V> implements KeyValueStore<K, V> {
}
@Override
+ public K peekNextKey() {
+ throw new UnsupportedOperationException("peekNextKey not supported");
+ }
+
+ @Override
public boolean hasNext() {
return underlying.hasNext();
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIteratorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIteratorTest.java
index dee2593..db391ed 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheKeyValueStoreIteratorTest.java
@@ -19,6 +19,7 @@ package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.StateSerdes;
import org.junit.Before;
@@ -50,7 +51,7 @@ public class MergedSortedCacheKeyValueStoreIteratorTest {
final Bytes from = Bytes.wrap(new byte[]{2});
final Bytes to = Bytes.wrap(new byte[]{9});
- final PeekingKeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>(store.range(from, to));
+ final KeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>("store", store.range(from, to));
final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.range(namespace, from.get(), to.get());
final MergedSortedCacheKeyValueStoreIterator<byte[], byte[]> iterator = new MergedSortedCacheKeyValueStoreIterator<>(cacheIterator, storeIterator, serdes);
@@ -138,9 +139,40 @@ public class MergedSortedCacheKeyValueStoreIteratorTest {
}
+ @Test
+ public void shouldPeekNextKey() throws Exception {
+ final KeyValueStore<Bytes, byte[]> kv = new InMemoryKeyValueStore<>("one");
+ final ThreadCache cache = new ThreadCache(1000000L);
+ byte[][] bytes = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}};
+ final String namespace = "one";
+ for (int i = 0; i < bytes.length - 1; i += 2) {
+ kv.put(Bytes.wrap(bytes[i]), bytes[i]);
+ cache.put(namespace, bytes[i + 1], new LRUCacheEntry(bytes[i + 1]));
+ }
+
+ final Bytes from = Bytes.wrap(new byte[]{2});
+ final Bytes to = Bytes.wrap(new byte[]{9});
+ final KeyValueIterator<Bytes, byte[]> storeIterator = kv.range(from, to);
+ final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.range(namespace, from.get(), to.get());
+
+ final MergedSortedCacheKeyValueStoreIterator<byte[], byte[]> iterator =
+ new MergedSortedCacheKeyValueStoreIterator<>(cacheIterator,
+ storeIterator,
+ serdes);
+ final byte[][] values = new byte[8][];
+ int index = 0;
+ int bytesIndex = 2;
+ while (iterator.hasNext()) {
+ final byte[] keys = iterator.peekNextKey();
+ values[index++] = keys;
+ assertArrayEquals(bytes[bytesIndex++], keys);
+ iterator.next();
+ }
+ }
+
private MergedSortedCacheKeyValueStoreIterator<byte[], byte[]> createIterator() {
final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.all(namespace);
- final PeekingKeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>(store.all());
+ final KeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>("store", store.all());
return new MergedSortedCacheKeyValueStoreIterator<>(cacheIterator, storeIterator, serdes);
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWindowStoreIteratorTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWindowStoreIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWindowStoreIteratorTest.java
index 9ee8b29..c33f174 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWindowStoreIteratorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MergedSortedCacheWindowStoreIteratorTest.java
@@ -18,13 +18,14 @@
package org.apache.kafka.streams.state.internals;
import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.StateSerdes;
-import org.apache.kafka.streams.state.WindowStoreIterator;
+import org.apache.kafka.test.KeyValueIteratorStub;
import org.junit.Test;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import static org.junit.Assert.assertArrayEquals;
@@ -34,16 +35,17 @@ public class MergedSortedCacheWindowStoreIteratorTest {
@Test
public void shouldIterateOverValueFromBothIterators() throws Exception {
- final List<KeyValue<Long, byte[]>> storeValues = new ArrayList<>();
+ final List<KeyValue<Bytes, byte[]>> storeValues = new ArrayList<>();
final ThreadCache cache = new ThreadCache(1000000L);
final String namespace = "one";
final StateSerdes<String, String> stateSerdes = new StateSerdes<>("foo", Serdes.String(), Serdes.String());
final List<KeyValue<Long, byte[]>> expectedKvPairs = new ArrayList<>();
for (long t = 0; t < 100; t += 20) {
- final KeyValue<Long, byte[]> v1 = KeyValue.pair(t, String.valueOf(t).getBytes());
+ final byte[] v1Bytes = String.valueOf(t).getBytes();
+ final KeyValue<Bytes, byte[]> v1 = KeyValue.pair(Bytes.wrap(WindowStoreUtils.toBinaryKey("a", t, 0, stateSerdes)), v1Bytes);
storeValues.add(v1);
- expectedKvPairs.add(v1);
+ expectedKvPairs.add(KeyValue.pair(t, v1Bytes));
final byte[] keyBytes = WindowStoreUtils.toBinaryKey("a", t + 10, 0, stateSerdes);
final byte[] valBytes = String.valueOf(t + 10).getBytes();
expectedKvPairs.add(KeyValue.pair(t + 10, valBytes));
@@ -52,11 +54,11 @@ public class MergedSortedCacheWindowStoreIteratorTest {
byte[] binaryFrom = WindowStoreUtils.toBinaryKey("a", 0, 0, stateSerdes);
byte[] binaryTo = WindowStoreUtils.toBinaryKey("a", 100, 0, stateSerdes);
- final PeekingWindowIterator<byte[]> storeIterator = new DelegatingPeekingWindowIterator<>(new WindowStoreIteratorStub(storeValues.iterator()));
+ final KeyValueIterator<Bytes, byte[]> storeIterator = new DelegatingPeekingKeyValueIterator<>("name", new KeyValueIteratorStub<>(storeValues.iterator()));
final ThreadCache.MemoryLRUCacheBytesIterator cacheIterator = cache.range(namespace, binaryFrom, binaryTo);
- final MergedSortedCachedWindowStoreIterator<byte[], byte[]> iterator = new MergedSortedCachedWindowStoreIterator<>(cacheIterator, storeIterator, new StateSerdes<>("name", Serdes.ByteArray(), Serdes.ByteArray()));
+ final MergedSortedCachedWindowStoreIterator<Bytes, byte[]> iterator = new MergedSortedCachedWindowStoreIterator<>(cacheIterator, storeIterator, new StateSerdes<>("name", Serdes.Bytes(), Serdes.ByteArray()));
int index = 0;
while (iterator.hasNext()) {
final KeyValue<Long, byte[]> next = iterator.next();
@@ -66,32 +68,4 @@ public class MergedSortedCacheWindowStoreIteratorTest {
}
}
- private static class WindowStoreIteratorStub implements WindowStoreIterator<byte[]> {
-
- private final Iterator<KeyValue<Long, byte[]>> iterator;
-
- public WindowStoreIteratorStub(final Iterator<KeyValue<Long, byte[]>> iterator) {
- this.iterator = iterator;
- }
-
- @Override
- public void close() {
- //no-op
- }
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public KeyValue<Long, byte[]> next() {
- return iterator.next();
- }
-
- @Override
- public void remove() {
-
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStoreTest.java
new file mode 100644
index 0000000..1587f13
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSegmentedBytesStoreTest.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.streams.StreamsMetrics;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.SegmentedBytesStoreStub;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertTrue;
+
+public class MeteredSegmentedBytesStoreTest {
+ private final SegmentedBytesStoreStub bytesStore = new SegmentedBytesStoreStub();
+ private final MeteredSegmentedBytesStore store = new MeteredSegmentedBytesStore(bytesStore, "scope", new MockTime());
+ private final Set<String> latencyRecorded = new HashSet<>();
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setUp() throws Exception {
+ final Metrics metrics = new Metrics();
+ final StreamsMetrics streamsMetrics = new StreamsMetrics() {
+
+ @Override
+ public Sensor addLatencySensor(final String scopeName, final String entityName, final String operationName, final String... tags) {
+ return metrics.sensor(operationName);
+ }
+
+ @Override
+ public void recordLatency(final Sensor sensor, final long startNs, final long endNs) {
+ latencyRecorded.add(sensor.name());
+ }
+ };
+
+ final MockProcessorContext context = new MockProcessorContext(null,
+ TestUtils.tempDirectory(),
+ Serdes.String(),
+ Serdes.Long(),
+ new NoOpRecordCollector(),
+ new ThreadCache(0)) {
+ @Override
+ public StreamsMetrics metrics() {
+ return streamsMetrics;
+ }
+ };
+ store.init(context, store);
+ }
+
+ @Test
+ public void shouldRecordRestoreLatencyOnInit() throws Exception {
+ assertTrue(latencyRecorded.contains("restore"));
+ assertTrue(bytesStore.initialized);
+ }
+
+ @Test
+ public void shouldRecordPutLatency() throws Exception {
+ store.put(Bytes.wrap(new byte[0]), new byte[0]);
+ assertTrue(latencyRecorded.contains("put"));
+ assertTrue(bytesStore.putCalled);
+ }
+
+ @Test
+ public void shouldRecordFetchLatency() throws Exception {
+ store.fetch(Bytes.wrap(new byte[0]), 1, 1).close(); // recorded on close;
+ assertTrue(latencyRecorded.contains("fetch"));
+ assertTrue(bytesStore.fetchCalled);
+ }
+
+ @Test
+ public void shouldRecordRemoveLatency() throws Exception {
+ store.remove(null);
+ assertTrue(latencyRecorded.contains("remove"));
+ assertTrue(bytesStore.removeCalled);
+ }
+
+ @Test
+ public void shouldRecordFlushLatency() throws Exception {
+ store.flush();
+ assertTrue(latencyRecorded.contains("flush"));
+ assertTrue(bytesStore.flushed);
+ }
+
+ @Test
+ public void shouldRecordGetLatency() throws Exception {
+ store.get(null);
+ assertTrue(latencyRecorded.contains("get"));
+ assertTrue(bytesStore.getCalled);
+ }
+
+ @Test
+ public void shouldCloseUnderlyingStore() throws Exception {
+ store.close();
+ assertTrue(bytesStore.closed);
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
index 0a782d5..99deb50 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/NamedCacheTest.java
@@ -31,6 +31,7 @@ import java.util.List;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
@@ -200,6 +201,21 @@ public class NamedCacheTest {
}
@Test
+ public void shouldRemoveDeletedValuesOnFlush() throws Exception {
+ cache.setListener(new ThreadCache.DirtyEntryFlushListener() {
+ @Override
+ public void apply(final List<ThreadCache.DirtyEntry> dirty) {
+ // no-op
+ }
+ });
+ cache.put(Bytes.wrap(new byte[]{0}), new LRUCacheEntry(null, true, 0, 0, 0, ""));
+ cache.put(Bytes.wrap(new byte[]{1}), new LRUCacheEntry(new byte[]{20}, true, 0, 0, 0, ""));
+ cache.flush();
+ assertEquals(1, cache.size());
+ assertNotNull(cache.get(Bytes.wrap(new byte[]{1})));
+ }
+
+ @Test
public void shouldBeReentrantAndNotBreakLRU() throws Exception {
final LRUCacheEntry dirty = new LRUCacheEntry(new byte[]{3}, true, 0, 0, 0, "");
final LRUCacheEntry clean = new LRUCacheEntry(new byte[]{3});
@@ -259,4 +275,5 @@ public class NamedCacheTest {
cache.put(key, dirty);
cache.evict();
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java
new file mode 100644
index 0000000..d4c81c3
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSegmentedBytesStoreTest.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.SessionKeySerde;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class RocksDBSegmentedBytesStoreTest {
+
+ private final long retention = 60000L;
+ private final int numSegments = 3;
+ private final String storeName = "bytes-store";
+ private RocksDBSegmentedBytesStore bytesStore;
+ private File stateDir;
+
+ @Before
+ public void before() {
+
+ bytesStore = new RocksDBSegmentedBytesStore(storeName,
+ retention,
+ numSegments,
+ new SessionKeySchema());
+
+ stateDir = TestUtils.tempDirectory();
+ final MockProcessorContext context = new MockProcessorContext(null,
+ stateDir,
+ Serdes.String(),
+ Serdes.Long(),
+ new NoOpRecordCollector(),
+ new ThreadCache(0));
+ bytesStore.init(context, bytesStore);
+ }
+
+ @After
+ public void close() {
+ bytesStore.close();
+ }
+
+ @Test
+ public void shouldPutAndFetch() throws Exception {
+ final String key = "a";
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(10, 10L))), serializeValue(10L));
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(500L, 1000L))), serializeValue(50L));
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(1500L, 2000L))), serializeValue(100L));
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(2500L, 3000L))), serializeValue(200L));
+
+ final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(KeyValue.pair(new Windowed<>(key, new TimeWindow(10, 10)), 10L),
+ KeyValue.pair(new Windowed<>(key, new TimeWindow(500, 1000)), 50L));
+
+ final KeyValueIterator<Bytes, byte[]> values = bytesStore.fetch(Bytes.wrap(key.getBytes()), 0, 1000L);
+ assertEquals(expected, toList(values));
+ }
+
+ @Test
+ public void shouldFindValuesWithinRange() throws Exception {
+ final String key = "a";
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(0L, 0L))), serializeValue(50L));
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(1000L, 1000L))), serializeValue(10L));
+ final KeyValueIterator<Bytes, byte[]> results = bytesStore.fetch(Bytes.wrap(key.getBytes()), 1L, 1999L);
+ assertEquals(Collections.singletonList(KeyValue.pair(new Windowed<>(key, new TimeWindow(1000L, 1000L)), 10L)), toList(results));
+ }
+
+ @Test
+ public void shouldRemove() throws Exception {
+ bytesStore.put(serializeKey(new Windowed<>("a", new TimeWindow(0, 1000))), serializeValue(30L));
+ bytesStore.put(serializeKey(new Windowed<>("a", new TimeWindow(1500, 2500))), serializeValue(50L));
+
+ bytesStore.remove(serializeKey(new Windowed<>("a", new TimeWindow(0, 1000))));
+ final KeyValueIterator<Bytes, byte[]> value = bytesStore.fetch(Bytes.wrap("a".getBytes()), 0, 1000L);
+ assertFalse(value.hasNext());
+ }
+
+ @Test
+ public void shouldRollSegments() throws Exception {
+ // just to validate directories
+ final Segments segments = new Segments(storeName, retention, numSegments);
+ final String key = "a";
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(0L, 0L))), serializeValue(50L));
+ assertEquals(Collections.singleton(segments.segmentName(0)), segmentDirs());
+
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(30000L, 60000L))), serializeValue(100L));
+ assertEquals(Utils.mkSet(segments.segmentName(0),
+ segments.segmentName(1)), segmentDirs());
+
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(61000L, 120000L))), serializeValue(200L));
+ assertEquals(Utils.mkSet(segments.segmentName(0),
+ segments.segmentName(1),
+ segments.segmentName(2)), segmentDirs());
+
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(121000L, 180000L))), serializeValue(300L));
+ assertEquals(Utils.mkSet(segments.segmentName(1),
+ segments.segmentName(2),
+ segments.segmentName(3)), segmentDirs());
+
+ bytesStore.put(serializeKey(new Windowed<>(key, new TimeWindow(181000L, 240000L))), serializeValue(400L));
+ assertEquals(Utils.mkSet(segments.segmentName(2),
+ segments.segmentName(3),
+ segments.segmentName(4)), segmentDirs());
+
+ final List<KeyValue<Windowed<String>, Long>> results = toList(bytesStore.fetch(Bytes.wrap(key.getBytes()), 0, 240000));
+ assertEquals(Arrays.asList(KeyValue.pair(new Windowed<>(key, new TimeWindow(61000L, 120000L)), 200L),
+ KeyValue.pair(new Windowed<>(key, new TimeWindow(121000L, 180000L)), 300L),
+ KeyValue.pair(new Windowed<>(key, new TimeWindow(181000L, 240000L)), 400L)
+ ), results);
+
+ }
+
+ private Set<String> segmentDirs() {
+ File windowDir = new File(stateDir, storeName);
+
+ return new HashSet<>(Arrays.asList(windowDir.list()));
+ }
+
+ private byte[] serializeValue(final long value) {
+ return Serdes.Long().serializer().serialize("", value);
+ }
+
+ private Bytes serializeKey(final Windowed<String> key) {
+ return SessionKeySerde.toBinary(key, Serdes.String().serializer());
+ }
+
+ private List<KeyValue<Windowed<String>, Long>> toList(final KeyValueIterator<Bytes, byte[]> iterator) {
+ final List<KeyValue<Windowed<String>, Long>> results = new ArrayList<>();
+ while (iterator.hasNext()) {
+ final KeyValue<Bytes, byte[]> next = iterator.next();
+ final KeyValue<Windowed<String>, Long> deserialized
+ = KeyValue.pair(SessionKeySerde.from(next.key.get(), Serdes.String().deserializer()), Serdes.Long().deserializer().deserialize("", next.value));
+ results.add(deserialized);
+ }
+ return results;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kafka/blob/e0de3a42/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java
----------------------------------------------------------------------
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java
new file mode 100644
index 0000000..11766c7
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBSessionStoreTest.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.streams.state.internals;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.kstream.Windowed;
+import org.apache.kafka.streams.kstream.internals.TimeWindow;
+import org.apache.kafka.streams.state.KeyValueIterator;
+import org.apache.kafka.streams.state.SessionStore;
+import org.apache.kafka.test.MockProcessorContext;
+import org.apache.kafka.test.NoOpRecordCollector;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class RocksDBSessionStoreTest {
+
+ private SessionStore<String, Long> sessionStore;
+
+ @Before
+ public void before() {
+ final RocksDBSegmentedBytesStore bytesStore =
+ new RocksDBSegmentedBytesStore("session-store", 10000L, 3, new SessionKeySchema());
+
+ sessionStore = new RocksDBSessionStore<>(bytesStore,
+ Serdes.String(),
+ Serdes.Long());
+
+ final MockProcessorContext context = new MockProcessorContext(null,
+ TestUtils.tempDirectory(),
+ Serdes.String(),
+ Serdes.Long(),
+ new NoOpRecordCollector(),
+ new ThreadCache(0));
+ sessionStore.init(context, sessionStore);
+ }
+
+ @After
+ public void close() {
+ sessionStore.close();
+ }
+
+ @Test
+ public void shouldPutAndFindSessionsInRange() throws Exception {
+ final String key = "a";
+ final Windowed<String> a1 = new Windowed<>(key, new TimeWindow(10, 10L));
+ final Windowed<String> a2 = new Windowed<>(key, new TimeWindow(500L, 1000L));
+ sessionStore.put(a1, 1L);
+ sessionStore.put(a2, 2L);
+ sessionStore.put(new Windowed<>(key, new TimeWindow(1500L, 2000L)), 1L);
+ sessionStore.put(new Windowed<>(key, new TimeWindow(2500L, 3000L)), 2L);
+
+ final List<KeyValue<Windowed<String>, Long>> expected
+ = Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L));
+
+ final KeyValueIterator<Windowed<String>, Long> values = sessionStore.findSessionsToMerge(key, 0, 1000L);
+ assertEquals(expected, toList(values));
+ }
+
+ @Test
+ public void shouldFetchAllSessionsWithSameRecordKey() throws Exception {
+
+ final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(KeyValue.pair(new Windowed<>("a", new TimeWindow(0, 0)), 1L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(10, 10)), 2L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(100, 100)), 3L),
+ KeyValue.pair(new Windowed<>("a", new TimeWindow(1000, 1000)), 4L));
+ for (KeyValue<Windowed<String>, Long> kv : expected) {
+ sessionStore.put(kv.key, kv.value);
+ }
+
+ // add one that shouldn't appear in the results
+ sessionStore.put(new Windowed<>("aa", new TimeWindow(0, 0)), 5L);
+
+ final List<KeyValue<Windowed<String>, Long>> results = toList(sessionStore.fetch("a"));
+ assertEquals(expected, results);
+
+ }
+
+
+ @Test
+ public void shouldFindValuesWithinMergingSessionWindowRange() throws Exception {
+ final String key = "a";
+ sessionStore.put(new Windowed<>(key, new TimeWindow(0L, 0L)), 1L);
+ sessionStore.put(new Windowed<>(key, new TimeWindow(1000L, 1000L)), 2L);
+ final KeyValueIterator<Windowed<String>, Long> results = sessionStore.findSessionsToMerge(key, -1, 1000L);
+
+ final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(
+ KeyValue.pair(new Windowed<>(key, new TimeWindow(0L, 0L)), 1L),
+ KeyValue.pair(new Windowed<>(key, new TimeWindow(1000L, 1000L)), 2L));
+ assertEquals(expected, toList(results));
+ }
+
+ @Test
+ public void shouldRemove() throws Exception {
+ sessionStore.put(new Windowed<>("a", new TimeWindow(0, 1000)), 1L);
+ sessionStore.put(new Windowed<>("a", new TimeWindow(1500, 2500)), 2L);
+
+ sessionStore.remove(new Windowed<>("a", new TimeWindow(0, 1000)));
+ assertFalse(sessionStore.findSessionsToMerge("a", 0, 1000L).hasNext());
+
+ assertTrue(sessionStore.findSessionsToMerge("a", 1500, 2500).hasNext());
+ }
+
+ @Test
+ public void shouldFindSessionsToMerge() throws Exception {
+ final Windowed<String> session1 = new Windowed<>("a", new TimeWindow(0, 100));
+ final Windowed<String> session2 = new Windowed<>("a", new TimeWindow(101, 200));
+ final Windowed<String> session3 = new Windowed<>("a", new TimeWindow(201, 300));
+ final Windowed<String> session4 = new Windowed<>("a", new TimeWindow(301, 400));
+ final Windowed<String> session5 = new Windowed<>("a", new TimeWindow(401, 500));
+ sessionStore.put(session1, 1L);
+ sessionStore.put(session2, 2L);
+ sessionStore.put(session3, 3L);
+ sessionStore.put(session4, 4L);
+ sessionStore.put(session5, 5L);
+ final KeyValueIterator<Windowed<String>, Long> results = sessionStore.findSessionsToMerge("a", 150, 300);
+ assertEquals(session2, results.next().key);
+ assertEquals(session3, results.next().key);
+ assertFalse(results.hasNext());
+ }
+
+ static List<KeyValue<Windowed<String>, Long>> toList(final KeyValueIterator<Windowed<String>, Long> iterator) {
+ final List<KeyValue<Windowed<String>, Long>> results = new ArrayList<>();
+ while (iterator.hasNext()) {
+ results.add(iterator.next());
+ }
+ return results;
+ }
+
+
+}
\ No newline at end of file