You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by vv...@apache.org on 2021/01/07 04:18:09 UTC
[kafka] 01/01: KAFKA-10867: Improved task idling
This is an automated email from the ASF dual-hosted git repository.
vvcephei pushed a commit to branch kafka-10867-improved-task-idling
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 608335f96aa9ae22806da62bda7a2a4119130271
Author: John Roesler <vv...@apache.org>
AuthorDate: Mon Jan 4 16:31:09 2021 -0600
KAFKA-10867: Improved task idling
Use the new ConsumerRecords.metadata() API to implement
improved task idling as described in KIP-695
---
.../org/apache/kafka/streams/StreamsConfig.java | 4 +-
.../processor/internals/PartitionGroup.java | 136 ++++++-
.../streams/processor/internals/StandbyTask.java | 6 +
.../streams/processor/internals/StreamTask.java | 63 ++-
.../streams/processor/internals/StreamThread.java | 38 +-
.../kafka/streams/processor/internals/Task.java | 9 +
.../streams/processor/internals/TaskManager.java | 3 +-
.../processor/internals/PartitionGroupTest.java | 360 ++++++++++++++---
.../processor/internals/StreamTaskTest.java | 444 ++++++++++-----------
.../processor/internals/StreamThreadTest.java | 45 ++-
.../processor/internals/TaskManagerTest.java | 6 +
.../internals/testutil/LogCaptureAppender.java | 4 +
.../apache/kafka/streams/TopologyTestDriver.java | 32 +-
13 files changed, 796 insertions(+), 354 deletions(-)
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index ae0ef30..41d8673 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -134,6 +134,8 @@ import static org.apache.kafka.common.config.ConfigDef.ValidString.in;
@SuppressWarnings("deprecation")
public class StreamsConfig extends AbstractConfig {
+ public static final long MAX_TASK_IDLE_MS_DISABLED = -1;
+
private final static Logger log = LoggerFactory.getLogger(StreamsConfig.class);
private static final ConfigDef CONFIG;
@@ -662,7 +664,7 @@ public class StreamsConfig extends AbstractConfig {
DEFAULT_WINDOWED_VALUE_SERDE_INNER_CLASS_DOC)
.define(MAX_TASK_IDLE_MS_CONFIG,
Type.LONG,
- 0L,
+ MAX_TASK_IDLE_MS_DISABLED,
Importance.MEDIUM,
MAX_TASK_IDLE_MS_DOC)
.define(MAX_WARMUP_REPLICAS_CONFIG,
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java
index 559e3b1..419aa5c 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java
@@ -17,16 +17,22 @@
package org.apache.kafka.streams.processor.internals;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.processor.TaskId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Set;
-import java.util.Iterator;
-import java.util.HashSet;
import java.util.function.Function;
/**
@@ -52,15 +58,20 @@ import java.util.function.Function;
* (i.e., it increases or stays the same over time).
*/
public class PartitionGroup {
+ private static final Logger LOG = LoggerFactory.getLogger(PartitionGroup.class);
private final Map<TopicPartition, RecordQueue> partitionQueues;
+ private final Sensor enforcedProcessingSensor;
+ private final long maxTaskIdleMs;
private final Sensor recordLatenessSensor;
+ private final TaskId id;
private final PriorityQueue<RecordQueue> nonEmptyQueuesByTime;
private long streamTime;
private int totalBuffered;
private boolean allBuffered;
-
+ private final Map<TopicPartition, Long> fetchedLags = new HashMap<>();
+ private final Map<TopicPartition, Long> idlePartitionDeadlines = new HashMap<>();
static class RecordInfo {
RecordQueue queue;
@@ -78,15 +89,130 @@ public class PartitionGroup {
}
}
- PartitionGroup(final Map<TopicPartition, RecordQueue> partitionQueues, final Sensor recordLatenessSensor) {
+ PartitionGroup(final TaskId id,
+ final Map<TopicPartition, RecordQueue> partitionQueues,
+ final Sensor recordLatenessSensor,
+ final Sensor enforcedProcessingSensor,
+ final long maxTaskIdleMs) {
+ this.id = id;
nonEmptyQueuesByTime = new PriorityQueue<>(partitionQueues.size(), Comparator.comparingLong(RecordQueue::headRecordTimestamp));
this.partitionQueues = partitionQueues;
+ this.enforcedProcessingSensor = enforcedProcessingSensor;
+ this.maxTaskIdleMs = maxTaskIdleMs;
this.recordLatenessSensor = recordLatenessSensor;
totalBuffered = 0;
allBuffered = false;
streamTime = RecordQueue.UNKNOWN;
}
+ public void addFetchedMetadata(final TopicPartition partition, final ConsumerRecords.Metadata metadata) {
+ fetchedLags.put(partition, metadata.lag());
+ }
+
+ public boolean readyToProcess(final long wallClockTime) {
+ // Log-level strategy:
+ // TRACE for messages that don't wait for fetches, since these may be logged at extremely high frequency
+ // DEBUG when we waited for a fetch and decided to wait some more, as configured
+ // DEBUG when we are ready for processing and didn't have to enforce processing
+ // INFO when we enforce processing, since this has to wait for fetches AND may result in disorder
+
+ if (maxTaskIdleMs == StreamsConfig.MAX_TASK_IDLE_MS_DISABLED) {
+ if (LOG.isTraceEnabled() && !allBuffered) {
+ final Set<TopicPartition> bufferedPartitions = new HashSet<>();
+ final Set<TopicPartition> emptyPartitions = new HashSet<>();
+ for (final Map.Entry<TopicPartition, RecordQueue> entry : partitionQueues.entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ emptyPartitions.add(entry.getKey());
+ } else {
+ bufferedPartitions.add(entry.getKey());
+ }
+ }
+ LOG.trace("[{}] Ready for processing because max.task.idle.ms is disabled." +
+ "\n\tThere may be out-of-order processing for this task as a result." +
+ "\n\tBuffered partitions: {}" +
+ "\n\tNon-buffered partitions: {}",
+ id,
+ bufferedPartitions,
+ emptyPartitions);
+ }
+ return true;
+ }
+
+ final Set<TopicPartition> queued = new HashSet<>();
+ Map<TopicPartition, Long> enforced = null;
+
+ for (final Map.Entry<TopicPartition, RecordQueue> entry : partitionQueues.entrySet()) {
+ final TopicPartition partition = entry.getKey();
+ final RecordQueue queue = entry.getValue();
+
+ final Long nullableFetchedLag = fetchedLags.remove(partition);
+
+ if (!queue.isEmpty()) {
+ // this partition is ready for processing
+ idlePartitionDeadlines.remove(partition);
+ queued.add(partition);
+ } else if (nullableFetchedLag == null) {
+ // must wait to fetch metadata for the partition
+ idlePartitionDeadlines.remove(partition);
+ LOG.trace("[{}] Waiting to fetch data for {}", id, partition);
+ return false;
+ } else if (nullableFetchedLag > 0L) {
+ // must wait to poll the data we know to be on the broker
+ idlePartitionDeadlines.remove(partition);
+ LOG.debug(
+ "[{}] Lag for {} is currently {}, but no data is buffered locally. Waiting to buffer some records.",
+ id,
+ partition,
+ nullableFetchedLag
+ );
+ return false;
+ } else {
+ // p is known to have zero lag. wait for maxTaskIdleMs to see if more data shows up.
+ // One alternative would be to set the deadline to nullableMetadata.receivedTimestamp + maxTaskIdleMs
+ // instead. That way, we would start the idling timer as of the freshness of our knowledge about the zero
+ // lag instead of when we happen to run this method, but realistically it's probably a small difference
+ // and using wall clock time seems more intuitive for users,
+ // since the log message will be as of wallClockTime.
+ idlePartitionDeadlines.putIfAbsent(partition, wallClockTime + maxTaskIdleMs);
+ final long deadline = idlePartitionDeadlines.get(partition);
+ if (wallClockTime < deadline) {
+ LOG.debug(
+ "[{}] Lag for {} is currently 0 and current time is {}. Waiting for new data to be produced for configured idle time {} (deadline is {}).",
+ id,
+ partition,
+ wallClockTime,
+ maxTaskIdleMs,
+ deadline
+ );
+ return false;
+ } else {
+ // this partition is ready for processing due to the task idling deadline passing
+ if (enforced == null) {
+ enforced = new HashMap<>();
+ }
+ enforced.put(partition, deadline);
+ }
+ }
+ }
+ if (enforced == null) {
+ LOG.debug("[{}] All partitions were buffered locally, so this task is ready for processing.", id);
+ } else {
+ enforcedProcessingSensor.record(1.0d, wallClockTime);
+ LOG.info("[{}] Continuing to process although some partition timestamps were not buffered locally." +
+ "\n\tThere may be out-of-order processing for this task as a result." +
+ "\n\tPartitions with local data: {}." +
+ "\n\tPartitions we gave up waiting for, with their corresponding deadlines: {}." +
+ "\n\tConfigured max.task.idle.ms: {}." +
+ "\n\tCurrent wall-clock time: {}.",
+ id,
+ queued,
+ enforced,
+ maxTaskIdleMs,
+ wallClockTime);
+ }
+ return true;
+ }
+
// visible for testing
long partitionTimestamp(final TopicPartition partition) {
final RecordQueue queue = partitionQueues.get(partition);
@@ -239,7 +365,7 @@ public class PartitionGroup {
return totalBuffered;
}
- boolean allPartitionsBuffered() {
+ boolean allPartitionsBufferedLocally() {
return allBuffered;
}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
index d052024..e91d2ea 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java
@@ -17,6 +17,7 @@
package org.apache.kafka.streams.processor.internals;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.metrics.Sensor;
@@ -286,6 +287,11 @@ public class StandbyTask extends AbstractTask implements Task {
throw new IllegalStateException("Attempted to add records to task " + id() + " for invalid input partition " + partition);
}
+ @Override
+ public void addFetchedMetadata(final TopicPartition partition, final ConsumerRecords.Metadata metadata) {
+ throw new IllegalStateException("Attempted to update metadata for standby task " + id());
+ }
+
InternalProcessorContext processorContext() {
return processorContext;
}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
index c4aa4b3..aab7857 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java
@@ -19,6 +19,7 @@ package org.apache.kafka.streams.processor.internals;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.TopicPartition;
@@ -78,7 +79,6 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
// leaked into this class, which is to checkpoint after committing if EOS is not enabled.
private final boolean eosEnabled;
- private final long maxTaskIdleMs;
private final int maxBufferedSize;
private final PartitionGroup partitionGroup;
private final RecordCollector recordCollector;
@@ -95,12 +95,10 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
private final Sensor processLatencySensor;
private final Sensor punctuateLatencySensor;
private final Sensor bufferedRecordsSensor;
- private final Sensor enforcedProcessingSensor;
private final Map<String, Sensor> e2eLatencySensors = new HashMap<>();
private final InternalProcessorContext processorContext;
private final RecordQueueCreator recordQueueCreator;
- private long idleStartTimeMs;
private boolean commitNeeded = false;
private boolean commitRequested = false;
@@ -139,12 +137,6 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
this.streamsMetrics = streamsMetrics;
closeTaskSensor = ThreadMetrics.closeTaskSensor(threadId, streamsMetrics);
final String taskId = id.toString();
- if (streamsMetrics.version() == Version.FROM_0100_TO_24) {
- final Sensor parent = ThreadMetrics.commitOverTasksSensor(threadId, streamsMetrics);
- enforcedProcessingSensor = TaskMetrics.enforcedProcessingSensor(threadId, taskId, streamsMetrics, parent);
- } else {
- enforcedProcessingSensor = TaskMetrics.enforcedProcessingSensor(threadId, taskId, streamsMetrics);
- }
processRatioSensor = TaskMetrics.activeProcessRatioSensor(threadId, taskId, streamsMetrics);
processLatencySensor = TaskMetrics.processLatencySensor(threadId, taskId, streamsMetrics);
punctuateLatencySensor = TaskMetrics.punctuateSensor(threadId, taskId, streamsMetrics);
@@ -167,7 +159,6 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
streamTimePunctuationQueue = new PunctuationQueue();
systemTimePunctuationQueue = new PunctuationQueue();
- maxTaskIdleMs = config.getLong(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG);
maxBufferedSize = config.getInt(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG);
// initialize the consumed and committed offset cache
@@ -176,8 +167,22 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
recordQueueCreator = new RecordQueueCreator(logContext, config.defaultTimestampExtractor(), config.defaultDeserializationExceptionHandler());
recordInfo = new PartitionGroup.RecordInfo();
- partitionGroup = new PartitionGroup(createPartitionQueues(),
- TaskMetrics.recordLatenessSensor(threadId, taskId, streamsMetrics));
+
+ final Sensor enforcedProcessingSensor;
+ if (streamsMetrics.version() == Version.FROM_0100_TO_24) {
+ final Sensor parent = ThreadMetrics.commitOverTasksSensor(threadId, streamsMetrics);
+ enforcedProcessingSensor = TaskMetrics.enforcedProcessingSensor(threadId, taskId, streamsMetrics, parent);
+ } else {
+ enforcedProcessingSensor = TaskMetrics.enforcedProcessingSensor(threadId, taskId, streamsMetrics);
+ }
+ final long maxTaskIdleMs = config.getLong(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG);
+ partitionGroup = new PartitionGroup(
+ id,
+ createPartitionQueues(),
+ TaskMetrics.recordLatenessSensor(threadId, taskId, streamsMetrics),
+ enforcedProcessingSensor,
+ maxTaskIdleMs
+ );
stateMgr.registerGlobalStateStores(topology.globalStateStores());
}
@@ -233,7 +238,6 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
initializeMetadata();
initializeTopology();
processorContext.initialize();
- idleStartTimeMs = RecordQueue.UNKNOWN;
transitionTo(State.RUNNING);
@@ -621,7 +625,12 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
/**
* An active task is processable if its buffer contains data for all of its input
- * source topic partitions, or if it is enforced to be processable
+ * source topic partitions, or if it is enforced to be processable.
+ *
+ * Note that this method is _NOT_ idempotent, because the internal bookkeeping
+ * consumes the partition metadata. For example, unit tests may have to invoke
+ * {@link #addFetchedMetadata(TopicPartition, ConsumerRecords.Metadata)} again
+ * invoking this method.
*/
public boolean isProcessable(final long wallClockTime) {
if (state() == State.CLOSED) {
@@ -633,26 +642,7 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
return false;
}
- if (partitionGroup.allPartitionsBuffered()) {
- idleStartTimeMs = RecordQueue.UNKNOWN;
- return true;
- } else if (partitionGroup.numBuffered() > 0) {
- if (idleStartTimeMs == RecordQueue.UNKNOWN) {
- idleStartTimeMs = wallClockTime;
- }
-
- if (wallClockTime - idleStartTimeMs >= maxTaskIdleMs) {
- enforcedProcessingSensor.record(1.0d, wallClockTime);
- return true;
- } else {
- return false;
- }
- } else {
- // there's no data in any of the topics; we should reset the enforced
- // processing timer
- idleStartTimeMs = RecordQueue.UNKNOWN;
- return false;
- }
+ return partitionGroup.readyToProcess(wallClockTime);
}
/**
@@ -901,6 +891,11 @@ public class StreamTask extends AbstractTask implements ProcessorNodePunctuator,
}
}
+ @Override
+ public void addFetchedMetadata(final TopicPartition partition, final ConsumerRecords.Metadata metadata) {
+ partitionGroup.addFetchedMetadata(partition, metadata);
+ }
+
/**
* Schedules a punctuation for the processor
*
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 0d6a42ba..c13aca2 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -331,6 +331,21 @@ public class StreamThread extends Thread {
final ThreadCache cache = new ThreadCache(logContext, cacheSizeBytes, streamsMetrics);
+ log.info("Creating consumer client");
+ final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG);
+ final Map<String, Object> consumerConfigs = config.getMainConsumerConfigs(applicationId, getConsumerClientId(threadId), threadIdx);
+ consumerConfigs.put(StreamsConfig.InternalConfig.REFERENCE_CONTAINER_PARTITION_ASSIGNOR, referenceContainer);
+
+ final String originalReset = (String) consumerConfigs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG);
+ // If there are any overrides, we never fall through to the consumer, but only handle offset management ourselves.
+ if (!builder.latestResetTopicsPattern().pattern().isEmpty() || !builder.earliestResetTopicsPattern().pattern().isEmpty()) {
+ consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
+ }
+
+ final int maxPollTimeMs =
+ new InternalConsumerConfig(config.getMainConsumerConfigs("", "", 0))
+ .getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG);
+
final ActiveTaskCreator activeTaskCreator = new ActiveTaskCreator(
builder,
config,
@@ -363,21 +378,10 @@ public class StreamThread extends Thread {
builder,
adminClient,
stateDirectory,
- StreamThread.processingMode(config)
+ processingMode(config)
);
referenceContainer.taskManager = taskManager;
- log.info("Creating consumer client");
- final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG);
- final Map<String, Object> consumerConfigs = config.getMainConsumerConfigs(applicationId, getConsumerClientId(threadId), threadIdx);
- consumerConfigs.put(StreamsConfig.InternalConfig.REFERENCE_CONTAINER_PARTITION_ASSIGNOR, referenceContainer);
-
- final String originalReset = (String) consumerConfigs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG);
- // If there are any overrides, we never fall through to the consumer, but only handle offset management ourselves.
- if (!builder.latestResetTopicsPattern().pattern().isEmpty() || !builder.earliestResetTopicsPattern().pattern().isEmpty()) {
- consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
- }
-
final Consumer<byte[], byte[]> mainConsumer = clientSupplier.getConsumer(consumerConfigs);
changelogReader.setMainConsumer(mainConsumer);
taskManager.setMainConsumer(mainConsumer);
@@ -400,7 +404,8 @@ public class StreamThread extends Thread {
referenceContainer.nextScheduledRebalanceMs,
shutdownErrorHook,
streamsUncaughtExceptionHandler,
- cacheSize -> cache.resize(cacheSize)
+ cacheSize -> cache.resize(cacheSize),
+ maxPollTimeMs
);
taskManager.setPartitionResetter(partitions -> streamThread.resetOffsets(partitions, null));
@@ -454,7 +459,8 @@ public class StreamThread extends Thread {
final AtomicLong nextProbingRebalanceMs,
final Runnable shutdownErrorHook,
final java.util.function.Consumer<Throwable> streamsUncaughtExceptionHandler,
- final java.util.function.Consumer<Long> cacheResizer) {
+ final java.util.function.Consumer<Long> cacheResizer,
+ final int maxPollTimeMs) {
super(threadId);
this.stateLock = new Object();
@@ -503,9 +509,7 @@ public class StreamThread extends Thread {
this.nextProbingRebalanceMs = nextProbingRebalanceMs;
this.pollTime = Duration.ofMillis(config.getLong(StreamsConfig.POLL_MS_CONFIG));
- final int dummyThreadIdx = 1;
- this.maxPollTimeMs = new InternalConsumerConfig(config.getMainConsumerConfigs("dummyGroupId", "dummyClientId", dummyThreadIdx))
- .getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG);
+ this.maxPollTimeMs = maxPollTimeMs;
this.commitTimeMs = config.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG);
this.numIterations = 1;
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
index 90aec7f..2033561 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Task.java
@@ -17,6 +17,7 @@
package org.apache.kafka.streams.processor.internals;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.TimeoutException;
@@ -121,8 +122,16 @@ public interface Task {
*/
void completeRestoration();
+ /**
+ * Add to this task any records returned from the poll.
+ */
void addRecords(TopicPartition partition, Iterable<ConsumerRecord<byte[], byte[]>> records);
+ /**
+ * Add to this task any metadata returned from the poll.
+ */
+ void addFetchedMetadata(TopicPartition partition, ConsumerRecords.Metadata metadata);
+
boolean commitNeeded();
/**
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 593717b..0f2fae9 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -1025,7 +1025,7 @@ public class TaskManager {
* @param records Records, can be null
*/
void addRecordsToTasks(final ConsumerRecords<byte[], byte[]> records) {
- for (final TopicPartition partition : records.partitions()) {
+ for (final TopicPartition partition : union(HashSet::new, records.partitions(), records.metadata().keySet())) {
final Task task = partitionToTask.get(partition);
if (task == null) {
@@ -1035,6 +1035,7 @@ public class TaskManager {
}
task.addRecords(partition, records.records(partition));
+ task.addFetchedMetadata(partition, records.metadata().get(partition));
}
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java
index 3425b30..ecfbef5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java
@@ -17,6 +17,7 @@
package org.apache.kafka.streams.processor.internals;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.metrics.Metrics;
@@ -29,33 +30,41 @@ import org.apache.kafka.common.serialization.Serializer;
import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.MockTime;
import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler;
+import org.apache.kafka.streams.processor.TaskId;
import org.apache.kafka.streams.processor.TimestampExtractor;
+import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
import org.apache.kafka.test.InternalMockProcessorContext;
import org.apache.kafka.test.MockSourceNode;
import org.apache.kafka.test.MockTimestampExtractor;
-import org.junit.Before;
+import org.hamcrest.Matchers;
import org.junit.Test;
import java.util.Arrays;
import java.util.List;
+import java.util.UUID;
import static org.apache.kafka.common.utils.Utils.mkEntry;
import static org.apache.kafka.common.utils.Utils.mkMap;
-import static org.hamcrest.CoreMatchers.is;
import static org.apache.kafka.common.utils.Utils.mkSet;
+import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class PartitionGroupTest {
+ private final TaskId id = new TaskId(0, 0);
+
+ private final long maxTaskIdleMs = StreamsConfig.MAX_TASK_IDLE_MS_DISABLED;
private final LogContext logContext = new LogContext();
private final Time time = new MockTime();
private final Serializer<Integer> intSerializer = new IntegerSerializer();
@@ -73,9 +82,9 @@ public class PartitionGroupTest {
private final byte[] recordKey = intSerializer.serialize(null, 1);
private final Metrics metrics = new Metrics();
+ private final Sensor enforcedProcessingSensor = metrics.sensor(UUID.randomUUID().toString());
private final MetricName lastLatenessValue = new MetricName("record-lateness-last-value", "", "", mkMap());
- private PartitionGroup group;
private static Sensor getValueSensor(final Metrics metrics, final MetricName metricName) {
final Sensor lastRecordedValue = metrics.sensor(metricName.name());
@@ -83,18 +92,12 @@ public class PartitionGroupTest {
return lastRecordedValue;
}
- @Before
- public void setUp() {
- group = new PartitionGroup(
- mkMap(mkEntry(partition1, queue1), mkEntry(partition2, queue2)),
- getValueSensor(metrics, lastLatenessValue)
- );
- }
-
@Test
public void testTimeTracking() {
- testFirstBatch();
- testSecondBatch();
+ final PartitionGroup group = getBasicGroup();
+
+ testFirstBatch(group);
+ testSecondBatch(group);
}
private RecordQueue createQueue1() {
@@ -127,7 +130,7 @@ public class PartitionGroupTest {
return new TopicPartition(topics[0], 2);
}
- private void testFirstBatch() {
+ private void testFirstBatch(final PartitionGroup group) {
StampedRecord record;
final PartitionGroup.RecordInfo info = new PartitionGroup.RecordInfo();
assertThat(group.numBuffered(), is(0));
@@ -151,7 +154,7 @@ public class PartitionGroupTest {
// 2:[2, 4, 6]
// st: -1 since no records was being processed yet
- verifyBuffered(6, 3, 3);
+ verifyBuffered(6, 3, 3, group);
assertThat(group.partitionTimestamp(partition1), is(RecordQueue.UNKNOWN));
assertThat(group.partitionTimestamp(partition2), is(RecordQueue.UNKNOWN));
assertThat(group.headRecordOffset(partition1), is(1L));
@@ -169,7 +172,7 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(RecordQueue.UNKNOWN));
assertThat(group.headRecordOffset(partition1), is(3L));
assertThat(group.headRecordOffset(partition2), is(2L));
- verifyTimes(record, 1L, 1L);
+ verifyTimes(record, 1L, 1L, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(0.0));
// get one record, now the time should be advanced
@@ -182,12 +185,12 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(2L));
assertThat(group.headRecordOffset(partition1), is(3L));
assertThat(group.headRecordOffset(partition2), is(4L));
- verifyTimes(record, 2L, 2L);
- verifyBuffered(4, 2, 2);
+ verifyTimes(record, 2L, 2L, group);
+ verifyBuffered(4, 2, 2, group);
assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
}
- private void testSecondBatch() {
+ private void testSecondBatch(final PartitionGroup group) {
StampedRecord record;
final PartitionGroup.RecordInfo info = new PartitionGroup.RecordInfo();
@@ -200,7 +203,7 @@ public class PartitionGroupTest {
// 1:[3, 5, 2, 4]
// 2:[4, 6]
// st: 2 (just adding records shouldn't change it)
- verifyBuffered(6, 4, 2);
+ verifyBuffered(6, 4, 2, group);
assertThat(group.partitionTimestamp(partition1), is(1L));
assertThat(group.partitionTimestamp(partition2), is(2L));
assertThat(group.headRecordOffset(partition1), is(3L));
@@ -218,8 +221,8 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(2L));
assertThat(group.headRecordOffset(partition1), is(5L));
assertThat(group.headRecordOffset(partition2), is(4L));
- verifyTimes(record, 3L, 3L);
- verifyBuffered(5, 3, 2);
+ verifyTimes(record, 3L, 3L, group);
+ verifyBuffered(5, 3, 2, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(0.0));
// get one record, time should be advanced
@@ -232,8 +235,8 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(4L));
assertThat(group.headRecordOffset(partition1), is(5L));
assertThat(group.headRecordOffset(partition2), is(6L));
- verifyTimes(record, 4L, 4L);
- verifyBuffered(4, 3, 1);
+ verifyTimes(record, 4L, 4L, group);
+ verifyBuffered(4, 3, 1, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(0.0));
// get one more record, time should be advanced
@@ -246,8 +249,8 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(4L));
assertThat(group.headRecordOffset(partition1), is(2L));
assertThat(group.headRecordOffset(partition2), is(6L));
- verifyTimes(record, 5L, 5L);
- verifyBuffered(3, 2, 1);
+ verifyTimes(record, 5L, 5L, group);
+ verifyBuffered(3, 2, 1, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(0.0));
// get one more record, time should not be advanced
@@ -260,8 +263,8 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(4L));
assertThat(group.headRecordOffset(partition1), is(4L));
assertThat(group.headRecordOffset(partition2), is(6L));
- verifyTimes(record, 2L, 5L);
- verifyBuffered(2, 1, 1);
+ verifyTimes(record, 2L, 5L, group);
+ verifyBuffered(2, 1, 1, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(3.0));
// get one more record, time should not be advanced
@@ -274,8 +277,8 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(4L));
assertNull(group.headRecordOffset(partition1));
assertThat(group.headRecordOffset(partition2), is(6L));
- verifyTimes(record, 4L, 5L);
- verifyBuffered(1, 0, 1);
+ verifyTimes(record, 4L, 5L, group);
+ verifyBuffered(1, 0, 1, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(1.0));
// get one more record, time should be advanced
@@ -288,13 +291,15 @@ public class PartitionGroupTest {
assertThat(group.partitionTimestamp(partition2), is(6L));
assertNull(group.headRecordOffset(partition1));
assertNull(group.headRecordOffset(partition2));
- verifyTimes(record, 6L, 6L);
- verifyBuffered(0, 0, 0);
+ verifyTimes(record, 6L, 6L, group);
+ verifyBuffered(0, 0, 0, group);
assertThat(metrics.metric(lastLatenessValue).metricValue(), is(0.0));
}
@Test
public void shouldChooseNextRecordBasedOnHeadTimestamp() {
+ final PartitionGroup group = getBasicGroup();
+
assertEquals(0, group.numBuffered());
// add three 3 records with timestamp 1, 5, 3 to partition-1
@@ -305,7 +310,7 @@ public class PartitionGroupTest {
group.addRawRecords(partition1, list1);
- verifyBuffered(3, 3, 0);
+ verifyBuffered(3, 3, 0, group);
assertEquals(-1L, group.streamTime());
assertEquals(0.0, metrics.metric(lastLatenessValue).metricValue());
@@ -341,12 +346,18 @@ public class PartitionGroupTest {
assertEquals(record.timestamp, 3L);
}
- private void verifyTimes(final StampedRecord record, final long recordTime, final long streamTime) {
+ private void verifyTimes(final StampedRecord record,
+ final long recordTime,
+ final long streamTime,
+ final PartitionGroup group) {
assertThat(record.timestamp, is(recordTime));
assertThat(group.streamTime(), is(streamTime));
}
- private void verifyBuffered(final int totalBuffered, final int partitionOneBuffered, final int partitionTwoBuffered) {
+ private void verifyBuffered(final int totalBuffered,
+ final int partitionOneBuffered,
+ final int partitionTwoBuffered,
+ final PartitionGroup group) {
assertEquals(totalBuffered, group.numBuffered());
assertEquals(partitionOneBuffered, group.numBuffered(partition1));
assertEquals(partitionTwoBuffered, group.numBuffered(partition2));
@@ -354,6 +365,8 @@ public class PartitionGroupTest {
@Test
public void shouldSetPartitionTimestampAndStreamTime() {
+ final PartitionGroup group = getBasicGroup();
+
group.setPartitionTime(partition1, 100L);
assertEquals(100L, group.partitionTimestamp(partition1));
assertEquals(100L, group.streamTime());
@@ -364,6 +377,8 @@ public class PartitionGroupTest {
@Test
public void shouldThrowIllegalStateExceptionUponAddRecordsIfPartitionUnknown() {
+ final PartitionGroup group = getBasicGroup();
+
final IllegalStateException exception = assertThrows(
IllegalStateException.class,
() -> group.addRawRecords(unknownPartition, null));
@@ -372,6 +387,8 @@ public class PartitionGroupTest {
@Test
public void shouldThrowIllegalStateExceptionUponNumBufferedIfPartitionUnknown() {
+ final PartitionGroup group = getBasicGroup();
+
final IllegalStateException exception = assertThrows(
IllegalStateException.class,
() -> group.numBuffered(unknownPartition));
@@ -380,6 +397,8 @@ public class PartitionGroupTest {
@Test
public void shouldThrowIllegalStateExceptionUponSetPartitionTimestampIfPartitionUnknown() {
+ final PartitionGroup group = getBasicGroup();
+
final IllegalStateException exception = assertThrows(
IllegalStateException.class,
() -> group.setPartitionTime(unknownPartition, 0L));
@@ -388,6 +407,8 @@ public class PartitionGroupTest {
@Test
public void shouldThrowIllegalStateExceptionUponGetPartitionTimestampIfPartitionUnknown() {
+ final PartitionGroup group = getBasicGroup();
+
final IllegalStateException exception = assertThrows(
IllegalStateException.class,
() -> group.partitionTimestamp(unknownPartition));
@@ -396,6 +417,8 @@ public class PartitionGroupTest {
@Test
public void shouldThrowIllegalStateExceptionUponGetHeadRecordOffsetIfPartitionUnknown() {
+ final PartitionGroup group = getBasicGroup();
+
final IllegalStateException exception = assertThrows(
IllegalStateException.class,
() -> group.headRecordOffset(unknownPartition));
@@ -404,6 +427,8 @@ public class PartitionGroupTest {
@Test
public void shouldEmptyPartitionsOnClear() {
+ final PartitionGroup group = getBasicGroup();
+
final List<ConsumerRecord<byte[], byte[]>> list = Arrays.asList(
new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
new ConsumerRecord<>("topic", 1, 3L, recordKey, recordValue),
@@ -424,6 +449,8 @@ public class PartitionGroupTest {
@Test
public void shouldUpdatePartitionQueuesShrink() {
+ final PartitionGroup group = getBasicGroup();
+
final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
@@ -434,7 +461,7 @@ public class PartitionGroupTest {
new ConsumerRecord<>("topic", 2, 6L, recordKey, recordValue));
group.addRawRecords(partition2, list2);
assertEquals(list1.size() + list2.size(), group.numBuffered());
- assertTrue(group.allPartitionsBuffered());
+ assertTrue(group.allPartitionsBufferedLocally());
group.nextRecord(new PartitionGroup.RecordInfo(), time.milliseconds());
// shrink list of queues
@@ -443,7 +470,7 @@ public class PartitionGroupTest {
return null;
});
- assertTrue(group.allPartitionsBuffered()); // because didn't add any new partitions
+ assertTrue(group.allPartitionsBufferedLocally()); // because didn't add any new partitions
assertEquals(list2.size(), group.numBuffered());
assertEquals(1, group.streamTime());
assertThrows(IllegalStateException.class, () -> group.partitionTimestamp(partition1));
@@ -453,9 +480,12 @@ public class PartitionGroupTest {
@Test
public void shouldUpdatePartitionQueuesExpand() {
- group = new PartitionGroup(
- mkMap(mkEntry(partition1, queue1)),
- getValueSensor(metrics, lastLatenessValue)
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(mkEntry(partition1, queue1)),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ maxTaskIdleMs
);
final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
@@ -463,7 +493,7 @@ public class PartitionGroupTest {
group.addRawRecords(partition1, list1);
assertEquals(list1.size(), group.numBuffered());
- assertTrue(group.allPartitionsBuffered());
+ assertTrue(group.allPartitionsBufferedLocally());
group.nextRecord(new PartitionGroup.RecordInfo(), time.milliseconds());
// expand list of queues
@@ -472,7 +502,7 @@ public class PartitionGroupTest {
return createQueue2();
});
- assertFalse(group.allPartitionsBuffered()); // because added new partition
+ assertFalse(group.allPartitionsBufferedLocally()); // because added new partition
assertEquals(1, group.numBuffered());
assertEquals(1, group.streamTime());
assertThat(group.partitionTimestamp(partition1), equalTo(1L));
@@ -482,16 +512,19 @@ public class PartitionGroupTest {
@Test
public void shouldUpdatePartitionQueuesShrinkAndExpand() {
- group = new PartitionGroup(
- mkMap(mkEntry(partition1, queue1)),
- getValueSensor(metrics, lastLatenessValue)
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(mkEntry(partition1, queue1)),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ maxTaskIdleMs
);
final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
group.addRawRecords(partition1, list1);
assertEquals(list1.size(), group.numBuffered());
- assertTrue(group.allPartitionsBuffered());
+ assertTrue(group.allPartitionsBufferedLocally());
group.nextRecord(new PartitionGroup.RecordInfo(), time.milliseconds());
// expand and shrink list of queues
@@ -500,11 +533,240 @@ public class PartitionGroupTest {
return createQueue2();
});
- assertFalse(group.allPartitionsBuffered()); // because added new partition
+ assertFalse(group.allPartitionsBufferedLocally()); // because added new partition
assertEquals(0, group.numBuffered());
assertEquals(1, group.streamTime());
assertThrows(IllegalStateException.class, () -> group.partitionTimestamp(partition1));
assertThat(group.partitionTimestamp(partition2), equalTo(RecordQueue.UNKNOWN));
assertThat(group.nextRecord(new PartitionGroup.RecordInfo(), time.milliseconds()), nullValue()); // all available records removed
}
+
+ @Test
+ public void shouldNeverWaitIfIdlingIsDisabled() {
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(
+ mkEntry(partition1, queue1),
+ mkEntry(partition2, queue2)
+ ),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ StreamsConfig.MAX_TASK_IDLE_MS_DISABLED
+ );
+
+ final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
+ new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
+ new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
+ group.addRawRecords(partition1, list1);
+
+ assertThat(group.allPartitionsBufferedLocally(), is(false));
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(0L), is(true));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("TRACE")),
+ Matchers.hasProperty("message", equalTo(
+ "[0_0] Ready for processing because max.task.idle.ms is disabled.\n" +
+ "\tThere may be out-of-order processing for this task as a result.\n" +
+ "\tBuffered partitions: [topic-1]\n" +
+ "\tNon-buffered partitions: [topic-2]"
+ ))
+ ))
+ );
+ }
+ }
+
+ @Test
+ public void shouldBeReadyIfAllPartitionsAreBuffered() {
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(
+ mkEntry(partition1, queue1),
+ mkEntry(partition2, queue2)
+ ),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ 0L
+ );
+
+ final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
+ new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
+ new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
+ group.addRawRecords(partition1, list1);
+
+ final List<ConsumerRecord<byte[], byte[]>> list2 = Arrays.asList(
+ new ConsumerRecord<>("topic", 2, 1L, recordKey, recordValue),
+ new ConsumerRecord<>("topic", 2, 5L, recordKey, recordValue));
+ group.addRawRecords(partition2, list2);
+
+ assertThat(group.allPartitionsBufferedLocally(), is(true));
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(0L), is(true));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("DEBUG")),
+ Matchers.hasProperty("message", equalTo("[0_0] All partitions were buffered locally, so this task is ready for processing."))
+ ))
+ );
+ }
+ }
+
+ @Test
+ public void shouldWaitForFetchesWhenMetadataIsIncomplete() {
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(
+ mkEntry(partition1, queue1),
+ mkEntry(partition2, queue2)
+ ),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ 0L
+ );
+
+ final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
+ new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
+ new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
+ group.addRawRecords(partition1, list1);
+
+ assertThat(group.allPartitionsBufferedLocally(), is(false));
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(0L), is(false));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("TRACE")),
+ Matchers.hasProperty("message", equalTo("[0_0] Waiting to fetch data for topic-2"))
+ ))
+ );
+ }
+ group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L, 0L));
+ assertThat(group.readyToProcess(0L), is(true));
+ }
+
+ @Test
+ public void shouldWaitForPollWhenLagIsNonzero() {
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(
+ mkEntry(partition1, queue1),
+ mkEntry(partition2, queue2)
+ ),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ 0L
+ );
+
+ final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
+ new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
+ new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
+ group.addRawRecords(partition1, list1);
+ group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L, 1L));
+
+ assertThat(group.allPartitionsBufferedLocally(), is(false));
+
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(0L), is(false));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("DEBUG")),
+ Matchers.hasProperty("message", equalTo("[0_0] Lag for topic-2 is currently 1, but no data is buffered locally. Waiting to buffer some records."))
+ ))
+ );
+ }
+ }
+
+ @Test
+ public void shouldIdleAsSpecifiedWhenLagIsZero() {
+ final PartitionGroup group = new PartitionGroup(
+ id,
+ mkMap(
+ mkEntry(partition1, queue1),
+ mkEntry(partition2, queue2)
+ ),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ 1L
+ );
+
+ final List<ConsumerRecord<byte[], byte[]>> list1 = Arrays.asList(
+ new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue),
+ new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue));
+ group.addRawRecords(partition1, list1);
+ group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L, 0L));
+
+ assertThat(group.allPartitionsBufferedLocally(), is(false));
+
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(0L), is(false));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("DEBUG")),
+ Matchers.hasProperty("message", equalTo("[0_0] Lag for topic-2 is currently 0 and current time is 0. Waiting for new data to be produced for configured idle time 1 (deadline is 1)."))
+ ))
+ );
+ }
+
+ group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L, 0L));
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(1L), is(true));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("INFO")),
+ Matchers.hasProperty("message", equalTo(
+ "[0_0] Continuing to process although some partition timestamps were not buffered locally.\n" +
+ "\tThere may be out-of-order processing for this task as a result.\n" +
+ "\tPartitions with local data: [topic-1].\n" +
+ "\tPartitions we gave up waiting for, with their corresponding deadlines: {topic-2=1}.\n" +
+ "\tConfigured max.task.idle.ms: 1.\n" +
+ "\tCurrent wall-clock time: 1."
+ ))
+ ))
+ );
+ }
+
+ group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L, 0L));
+ try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) {
+ LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class);
+ assertThat(group.readyToProcess(2L), is(true));
+ assertThat(
+ appender.getEvents(),
+ hasItem(Matchers.allOf(
+ Matchers.hasProperty("level", equalTo("INFO")),
+ Matchers.hasProperty("message", equalTo(
+ "[0_0] Continuing to process although some partition timestamps were not buffered locally.\n" +
+ "\tThere may be out-of-order processing for this task as a result.\n" +
+ "\tPartitions with local data: [topic-1].\n" +
+ "\tPartitions we gave up waiting for, with their corresponding deadlines: {topic-2=1}.\n" +
+ "\tConfigured max.task.idle.ms: 1.\n" +
+ "\tCurrent wall-clock time: 2."
+ ))
+ ))
+ );
+ }
+ }
+
+ private PartitionGroup getBasicGroup() {
+ return new PartitionGroup(
+ id,
+ mkMap(
+ mkEntry(partition1, queue1),
+ mkEntry(partition2, queue2)
+ ),
+ getValueSensor(metrics, lastLatenessValue),
+ enforcedProcessingSensor,
+ maxTaskIdleMs
+ );
+ }
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index a72f0fb..4c94c9b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -18,6 +18,7 @@ package org.apache.kafka.streams.processor.internals;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.MockConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
@@ -74,7 +75,6 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.Duration;
-import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashSet;
@@ -85,6 +85,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
@@ -183,10 +184,10 @@ public class StreamTaskTest {
final Set<String> repartitionTopics) {
return new ProcessorTopology(processorNodes,
sourcesByTopic,
- Collections.emptyMap(),
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyMap(),
+ emptyMap(),
+ emptyList(),
+ emptyList(),
+ emptyMap(),
repartitionTopics);
}
@@ -194,10 +195,10 @@ public class StreamTaskTest {
final Map<String, SourceNode<?, ?, ?, ?>> sourcesByTopic) {
return new ProcessorTopology(processorNodes,
sourcesByTopic,
- Collections.emptyMap(),
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyMap(),
+ emptyMap(),
+ emptyList(),
+ emptyList(),
+ emptyMap(),
Collections.emptySet());
}
@@ -281,7 +282,7 @@ public class StreamTaskTest {
EasyMock.expect(stateManager.taskType()).andStubReturn(TaskType.ACTIVE);
stateDirectory = ctrl.createMock(StateDirectory.class);
- stateManager.registerGlobalStateStores(Collections.emptyList());
+ stateManager.registerGlobalStateStores(emptyList());
EasyMock.expectLastCall();
EasyMock.expect(stateManager.taskId()).andReturn(taskId);
@@ -330,16 +331,16 @@ public class StreamTaskTest {
public void shouldTransitToRestoringThenRunningAfterCreation() throws IOException {
stateDirectory = EasyMock.createNiceMock(StateDirectory.class);
EasyMock.expect(stateDirectory.lock(taskId)).andReturn(true);
- EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.singleton(changelogPartition));
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.singletonMap(changelogPartition, 10L));
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(singleton(changelogPartition));
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(singletonMap(changelogPartition, 10L));
stateManager.registerStore(stateStore, stateStore.stateRestoreCallback);
EasyMock.expectLastCall();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateDirectory, stateManager, recordCollector);
task = createStatefulTask(createConfig(false, "100"), true);
- assertEquals(Task.State.CREATED, task.state());
+ assertEquals(CREATED, task.state());
task.initializeIfNeeded();
@@ -354,7 +355,7 @@ public class StreamTaskTest {
task.completeRestoration();
- assertEquals(Task.State.RUNNING, task.state());
+ assertEquals(RUNNING, task.state());
assertTrue(source1.initialized);
assertTrue(source2.initialized);
@@ -366,65 +367,82 @@ public class StreamTaskTest {
task = createStatelessTask(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 10),
- getConsumerRecord(partition1, 20),
- getConsumerRecord(partition1, 30)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 10, 101),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 20, 102),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 30, 103)
));
task.addRecords(partition2, asList(
- getConsumerRecord(partition2, 25),
- getConsumerRecord(partition2, 35),
- getConsumerRecord(partition2, 45)
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 25, 201),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 35, 202),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 45, 203)
));
assertTrue(task.process(0L));
assertEquals(5, task.numBuffered());
assertEquals(1, source1.numReceived);
assertEquals(0, source2.numReceived);
+ assertEquals(singletonList(101), source1.values);
+ assertEquals(emptyList(), source2.values);
assertTrue(task.process(0L));
assertEquals(4, task.numBuffered());
assertEquals(2, source1.numReceived);
assertEquals(0, source2.numReceived);
+ assertEquals(asList(101, 102), source1.values);
+ assertEquals(emptyList(), source2.values);
assertTrue(task.process(0L));
assertEquals(3, task.numBuffered());
assertEquals(2, source1.numReceived);
assertEquals(1, source2.numReceived);
+ assertEquals(asList(101, 102), source1.values);
+ assertEquals(singletonList(201), source2.values);
assertTrue(task.process(0L));
assertEquals(2, task.numBuffered());
assertEquals(3, source1.numReceived);
assertEquals(1, source2.numReceived);
+ assertEquals(asList(101, 102, 103), source1.values);
+ assertEquals(singletonList(201), source2.values);
+
+ // tell the task that it doesn't need to wait for more records on partition1
+ task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 30L, 0L, 30L));
assertTrue(task.process(0L));
assertEquals(1, task.numBuffered());
assertEquals(3, source1.numReceived);
assertEquals(2, source2.numReceived);
+ assertEquals(asList(101, 102, 103), source1.values);
+ assertEquals(asList(201, 202), source2.values);
+ // tell the task that it doesn't need to wait for more records on partition1
+ task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 30L, 0L, 30L));
assertTrue(task.process(0L));
assertEquals(0, task.numBuffered());
assertEquals(3, source1.numReceived);
assertEquals(3, source2.numReceived);
+ assertEquals(asList(101, 102, 103), source1.values);
+ assertEquals(asList(201, 202, 203), source2.values);
}
@Test
public void shouldRecordBufferedRecords() {
- task = createStatelessTask(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
+ task = createSingleSourceStateless(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
final KafkaMetric metric = getMetric("active-buffer", "%s-count", task.id().toString(), StreamsConfig.METRICS_LATEST);
assertThat(metric.metricValue(), equalTo(0.0));
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 10),
- getConsumerRecord(partition1, 20)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 10),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 20)
));
task.recordProcessTimeRatioAndBufferSize(100L, time.milliseconds());
assertThat(metric.metricValue(), equalTo(2.0));
- task.process(0L);
+ assertTrue(task.process(0L));
task.recordProcessTimeRatioAndBufferSize(100L, time.milliseconds());
assertThat(metric.metricValue(), equalTo(1.0));
@@ -493,7 +511,7 @@ public class StreamTaskTest {
final Metric terminalMax = getProcessorMetric("record-e2e-latency", "%s-max", task.id().toString(), terminalNodeName, StreamsConfig.METRICS_LATEST);
// e2e latency = 10
- task.addRecords(partition1, singletonList(getConsumerRecord(0, 0L)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(0, 0L)));
task.process(10L);
assertThat(sourceAvg.metricValue(), equalTo(10.0));
@@ -507,7 +525,7 @@ public class StreamTaskTest {
// e2e latency = 15
- task.addRecords(partition1, singletonList(getConsumerRecord(1, 0L)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(1, 0L)));
task.process(15L);
assertThat(sourceAvg.metricValue(), equalTo(12.5));
@@ -521,7 +539,7 @@ public class StreamTaskTest {
// e2e latency = 23
- task.addRecords(partition1, singletonList(getConsumerRecord(2, 0L)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(2, 0L)));
task.process(23L);
assertThat(sourceAvg.metricValue(), equalTo(16.0));
@@ -535,7 +553,7 @@ public class StreamTaskTest {
// e2e latency = 5
- task.addRecords(partition1, singletonList(getConsumerRecord(3, 0L)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(3, 0L)));
task.process(5L);
assertThat(sourceAvg.metricValue(), equalTo(13.25));
@@ -705,15 +723,15 @@ public class StreamTaskTest {
task = createStatelessTask(createConfig(false, "100"), StreamsConfig.METRICS_LATEST);
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 10),
- getConsumerRecord(partition1, 20)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 10),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 20)
));
task.addRecords(partition2, asList(
- getConsumerRecord(partition2, 35),
- getConsumerRecord(partition2, 45),
- getConsumerRecord(partition2, 55),
- getConsumerRecord(partition2, 65)
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 35),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 45),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 55),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 65)
));
assertTrue(task.process(0L));
@@ -724,9 +742,9 @@ public class StreamTaskTest {
assertTrue(consumer.paused().contains(partition2));
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 30),
- getConsumerRecord(partition1, 40),
- getConsumerRecord(partition1, 50)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 30),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 40),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 50)
));
assertEquals(2, consumer.paused().size());
@@ -761,17 +779,17 @@ public class StreamTaskTest {
task.completeRestoration();
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 20),
- getConsumerRecord(partition1, 142),
- getConsumerRecord(partition1, 155),
- getConsumerRecord(partition1, 160)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 20),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 142),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 155),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 160)
));
task.addRecords(partition2, asList(
- getConsumerRecord(partition2, 25),
- getConsumerRecord(partition2, 145),
- getConsumerRecord(partition2, 159),
- getConsumerRecord(partition2, 161)
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 25),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 145),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 159),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 161)
));
// st: -1
@@ -829,6 +847,9 @@ public class StreamTaskTest {
assertEquals(3, source2.numReceived);
assertTrue(task.maybePunctuateStreamTime());
+ // tell the task that it doesn't need to wait for new data on partition1
+ task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 160L, 0, 160L));
+
// st: 161
assertTrue(task.process(0L));
assertEquals(0, task.numBuffered());
@@ -846,15 +867,15 @@ public class StreamTaskTest {
task.completeRestoration();
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 20),
- getConsumerRecord(partition1, 30),
- getConsumerRecord(partition1, 40)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 20),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 30),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 40)
));
task.addRecords(partition2, asList(
- getConsumerRecord(partition2, 25),
- getConsumerRecord(partition2, 35),
- getConsumerRecord(partition2, 45)
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 25),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 35),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 45)
));
assertFalse(task.maybePunctuateStreamTime());
@@ -895,13 +916,13 @@ public class StreamTaskTest {
@Test
public void shouldRespectCommitNeeded() {
- task = createStatelessTask(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
+ task = createSingleSourceStateless(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
task.initializeIfNeeded();
task.completeRestoration();
assertFalse(task.commitNeeded());
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, 0)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, 0)));
assertTrue(task.process(0L));
assertTrue(task.commitNeeded());
@@ -933,14 +954,14 @@ public class StreamTaskTest {
@Test
public void shouldCommitNextOffsetFromQueueIfAvailable() {
- task = createStatelessTask(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
+ task = createSingleSourceStateless(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, Arrays.asList(
- getConsumerRecord(partition1, 0L),
- getConsumerRecord(partition1, 3L),
- getConsumerRecord(partition1, 5L)));
+ task.addRecords(partition1, asList(
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 0L),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 3L),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 5L)));
task.process(0L);
task.process(0L);
@@ -956,12 +977,14 @@ public class StreamTaskTest {
task.initializeIfNeeded();
task.completeRestoration();
- consumer.addRecord(getConsumerRecord(partition1, 0L));
- consumer.addRecord(getConsumerRecord(partition1, 1L));
- consumer.addRecord(getConsumerRecord(partition1, 2L));
+ consumer.addRecord(getConsumerRecordWithOffsetAsTimestamp(partition1, 0L));
+ consumer.addRecord(getConsumerRecordWithOffsetAsTimestamp(partition1, 1L));
+ consumer.addRecord(getConsumerRecordWithOffsetAsTimestamp(partition1, 2L));
+ consumer.updateEndOffsets(mkMap(mkEntry(partition2, 0L)));
consumer.poll(Duration.ZERO);
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, 0L)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, 0L)));
+ task.addRecords(partition2, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition2, 0L)));
task.process(0L);
final Map<TopicPartition, OffsetAndMetadata> offsetsAndMetadata = task.prepareCommit();
@@ -1024,116 +1047,16 @@ public class StreamTaskTest {
final byte[] bytes = ByteBuffer.allocate(4).putInt(1).array();
- task.addRecords(partition1, Collections.singleton(new ConsumerRecord<>(topic1, 1, 0, bytes, bytes)));
+ task.addRecords(partition1, singleton(new ConsumerRecord<>(topic1, 1, 0, bytes, bytes)));
assertFalse(task.process(0L));
- task.addRecords(partition2, Collections.singleton(new ConsumerRecord<>(topic2, 1, 0, bytes, bytes)));
+ task.addRecords(partition2, singleton(new ConsumerRecord<>(topic2, 1, 0, bytes, bytes)));
assertTrue(task.process(0L));
}
@Test
- public void shouldBeProcessableIfWaitedForTooLong() {
- // max idle time is 100ms
- task = createStatelessTask(createConfig(false, "100"), StreamsConfig.METRICS_LATEST);
- task.initializeIfNeeded();
- task.completeRestoration();
-
- final MetricName enforcedProcessMetric = metrics.metricName(
- "enforced-processing-total",
- "stream-task-metrics",
- mkMap(mkEntry("thread-id", Thread.currentThread().getName()), mkEntry("task-id", taskId.toString()))
- );
-
- assertFalse(task.process(time.milliseconds()));
- assertEquals(0.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- final byte[] bytes = ByteBuffer.allocate(4).putInt(1).array();
-
- task.addRecords(partition1,
- asList(
- new ConsumerRecord<>(topic1, 1, 0, bytes, bytes),
- new ConsumerRecord<>(topic1, 1, 1, bytes, bytes),
- new ConsumerRecord<>(topic1, 1, 2, bytes, bytes)
- )
- );
-
- assertFalse(task.process(time.milliseconds()));
-
- assertFalse(task.process(time.milliseconds() + 99L));
-
- assertTrue(task.process(time.milliseconds() + 100L));
- assertEquals(1.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- // once decided to enforce, continue doing that
- assertTrue(task.process(time.milliseconds() + 101L));
- assertEquals(2.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- task.addRecords(partition2, Collections.singleton(new ConsumerRecord<>(topic2, 1, 0, bytes, bytes)));
-
- assertTrue(task.process(time.milliseconds() + 130L));
- assertEquals(2.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- // one resumed to normal processing, the timer should be reset
-
- assertFalse(task.process(time.milliseconds() + 150L));
- assertEquals(2.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- assertFalse(task.process(time.milliseconds() + 249L));
- assertEquals(2.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- assertTrue(task.process(time.milliseconds() + 250L));
- assertEquals(3.0, metrics.metric(enforcedProcessMetric).metricValue());
- }
-
- @Test
- public void shouldNotBeProcessableIfNoDataAvailble() {
- task = createStatelessTask(createConfig(false, "100"), StreamsConfig.METRICS_LATEST);
- task.initializeIfNeeded();
- task.completeRestoration();
-
- final MetricName enforcedProcessMetric = metrics.metricName(
- "enforced-processing-total",
- "stream-task-metrics",
- mkMap(mkEntry("thread-id", Thread.currentThread().getName()), mkEntry("task-id", taskId.toString()))
- );
-
- assertFalse(task.process(0L));
- assertEquals(0.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- final byte[] bytes = ByteBuffer.allocate(4).putInt(1).array();
-
- task.addRecords(partition1, Collections.singleton(new ConsumerRecord<>(topic1, 1, 0, bytes, bytes)));
-
- assertFalse(task.process(time.milliseconds()));
-
- assertFalse(task.process(time.milliseconds() + 99L));
-
- assertTrue(task.process(time.milliseconds() + 100L));
- assertEquals(1.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- // once the buffer is drained and no new records coming, the timer should be reset
-
- assertFalse(task.process(time.milliseconds() + 110L));
- assertEquals(1.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- // check that after time is reset, we only falls into enforced processing after the
- // whole timeout has elapsed again
- task.addRecords(partition1, Collections.singleton(new ConsumerRecord<>(topic1, 1, 0, bytes, bytes)));
-
- assertFalse(task.process(time.milliseconds() + 150L));
- assertEquals(1.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- assertFalse(task.process(time.milliseconds() + 249L));
- assertEquals(1.0, metrics.metric(enforcedProcessMetric).metricValue());
-
- assertTrue(task.process(time.milliseconds() + 250L));
- assertEquals(2.0, metrics.metric(enforcedProcessMetric).metricValue());
- }
-
-
- @Test
public void shouldPunctuateSystemTimeWhenIntervalElapsed() {
task = createStatelessTask(createConfig(false, "100"), StreamsConfig.METRICS_LATEST);
task.initializeIfNeeded();
@@ -1243,8 +1166,8 @@ public class StreamTaskTest {
@Test
public void shouldWrapKafkaExceptionWithStreamsExceptionWhenProcess() {
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.emptyMap()).anyTimes();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
task = createFaultyStatefulTask(createConfig(false, "100"));
@@ -1253,14 +1176,14 @@ public class StreamTaskTest {
task.completeRestoration();
task.addRecords(partition1, asList(
- getConsumerRecord(partition1, 10),
- getConsumerRecord(partition1, 20),
- getConsumerRecord(partition1, 30)
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 10),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 20),
+ getConsumerRecordWithOffsetAsTimestamp(partition1, 30)
));
task.addRecords(partition2, asList(
- getConsumerRecord(partition2, 5), // this is the first record to process
- getConsumerRecord(partition2, 35),
- getConsumerRecord(partition2, 45)
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 5), // this is the first record to process
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 35),
+ getConsumerRecordWithOffsetAsTimestamp(partition2, 45)
));
assertThrows(StreamsException.class, () -> task.process(0L));
@@ -1271,7 +1194,7 @@ public class StreamTaskTest {
stateDirectory = EasyMock.createNiceMock(StateDirectory.class);
EasyMock.expect(stateDirectory.lock(taskId)).andReturn(true);
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(recordCollector, stateDirectory, stateManager);
@@ -1297,7 +1220,7 @@ public class StreamTaskTest {
task.suspend();
- assertEquals(Task.State.SUSPENDED, task.state());
+ assertEquals(SUSPENDED, task.state());
assertFalse(source1.initialized);
assertFalse(source2.initialized);
@@ -1309,14 +1232,14 @@ public class StreamTaskTest {
task.completeRestoration();
- assertEquals(Task.State.RUNNING, task.state());
+ assertEquals(RUNNING, task.state());
assertTrue(source1.initialized);
assertTrue(source2.initialized);
EasyMock.verify(stateManager, recordCollector);
EasyMock.reset(recordCollector);
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap());
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap());
EasyMock.replay(recordCollector);
}
@@ -1324,12 +1247,12 @@ public class StreamTaskTest {
public void shouldNotCheckpointOffsetsAgainOnCommitIfSnapshotNotChangedMuch() {
final Long offset = 543L;
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.singletonMap(changelogPartition, offset)).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(singletonMap(changelogPartition, offset)).anyTimes();
stateManager.checkpoint();
EasyMock.expectLastCall().once();
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(changelogPartition, 10L))
- .andReturn(Collections.singletonMap(changelogPartition, 20L));
+ .andReturn(singletonMap(changelogPartition, 10L))
+ .andReturn(singletonMap(changelogPartition, 20L));
EasyMock.expectLastCall();
EasyMock.replay(stateManager, recordCollector);
@@ -1351,14 +1274,14 @@ public class StreamTaskTest {
public void shouldCheckpointOffsetsOnCommitIfSnapshotMuchChanged() {
final Long offset = 543L;
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.singletonMap(changelogPartition, offset)).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(singletonMap(changelogPartition, offset)).anyTimes();
stateManager.checkpoint();
EasyMock.expectLastCall().times(2);
- EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.singleton(changelogPartition));
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(singleton(changelogPartition));
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(changelogPartition, 0L))
- .andReturn(Collections.singletonMap(changelogPartition, 10L))
- .andReturn(Collections.singletonMap(changelogPartition, 12000L));
+ .andReturn(singletonMap(changelogPartition, 0L))
+ .andReturn(singletonMap(changelogPartition, 10L))
+ .andReturn(singletonMap(changelogPartition, 12000L));
stateManager.registerStore(stateStore, stateStore.stateRestoreCallback);
EasyMock.expectLastCall();
EasyMock.replay(stateManager, recordCollector);
@@ -1378,10 +1301,10 @@ public class StreamTaskTest {
@Test
public void shouldNotCheckpointOffsetsOnCommitIfEosIsEnabled() {
- EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.singleton(changelogPartition));
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(singleton(changelogPartition));
stateManager.registerStore(stateStore, stateStore.stateRestoreCallback);
EasyMock.expectLastCall();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
task = createStatefulTask(createConfig(true, "100"), true);
@@ -1451,7 +1374,7 @@ public class StreamTaskTest {
EasyMock.expectLastCall();
stateManager.close();
EasyMock.expectLastCall();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
task = createFaultyStatefulTask(createConfig(false, "100"));
@@ -1472,13 +1395,13 @@ public class StreamTaskTest {
final ProcessorTopology topology = withRepartitionTopics(
asList(source1, source2),
mkMap(mkEntry(topic1, source1), mkEntry(repartition.topic(), source2)),
- Collections.singleton(repartition.topic())
+ singleton(repartition.topic())
);
consumer.assign(asList(partition1, repartition));
consumer.updateBeginningOffsets(mkMap(mkEntry(repartition, 0L)));
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet());
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
final StreamsConfig config = createConfig(false, "0");
@@ -1507,22 +1430,23 @@ public class StreamTaskTest {
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, 5L)));
- task.addRecords(repartition, singletonList(getConsumerRecord(repartition, 10L)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, 5L)));
+ task.addRecords(repartition, singletonList(getConsumerRecordWithOffsetAsTimestamp(repartition, 10L)));
assertTrue(task.process(0L));
+ task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 5L, 0L, 5L));
assertTrue(task.process(0L));
task.prepareCommit();
final Map<TopicPartition, Long> map = task.purgeableOffsets();
- assertThat(map, equalTo(Collections.singletonMap(repartition, 11L)));
+ assertThat(map, equalTo(singletonMap(repartition, 11L)));
}
@Test
public void shouldThrowStreamsExceptionWhenFetchCommittedFailed() {
- EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.singleton(partition1));
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(singleton(partition1));
EasyMock.replay(stateManager);
final Consumer<byte[], byte[]> consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {
@@ -1543,7 +1467,7 @@ public class StreamTaskTest {
public void shouldThrowIfCommittingOnIllegalState() {
task = createStatelessTask(createConfig(false, "100"), StreamsConfig.METRICS_LATEST);
- task.transitionTo(Task.State.SUSPENDED);
+ task.transitionTo(SUSPENDED);
task.transitionTo(Task.State.CLOSED);
assertThrows(IllegalStateException.class, task::prepareCommit);
}
@@ -1552,7 +1476,7 @@ public class StreamTaskTest {
public void shouldThrowIfPostCommittingOnIllegalState() {
task = createStatelessTask(createConfig(false, "100"), StreamsConfig.METRICS_LATEST);
- task.transitionTo(Task.State.SUSPENDED);
+ task.transitionTo(SUSPENDED);
task.transitionTo(Task.State.CLOSED);
assertThrows(IllegalStateException.class, () -> task.postCommit(true));
}
@@ -1573,7 +1497,7 @@ public class StreamTaskTest {
stateManager.checkpoint();
EasyMock.expectLastCall().once();
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(partition1, 1L));
+ .andReturn(singletonMap(partition1, 1L));
EasyMock.replay(stateManager);
task = createStatefulTask(createConfig(false, "100"), true);
@@ -1586,8 +1510,8 @@ public class StreamTaskTest {
@Test
public void shouldNotCheckpointForSuspendedRunningTaskWithSmallProgress() {
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(partition1, 1L))
- .andReturn(Collections.singletonMap(partition1, 2L));
+ .andReturn(singletonMap(partition1, 1L))
+ .andReturn(singletonMap(partition1, 2L));
stateManager.checkpoint();
EasyMock.expectLastCall().andThrow(new AssertionError("Checkpoint should not be called")).anyTimes();
EasyMock.replay(stateManager);
@@ -1607,8 +1531,8 @@ public class StreamTaskTest {
@Test
public void shouldCheckpointForSuspendedRunningTaskWithLargeProgress() {
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(partition1, 12000L))
- .andReturn(Collections.singletonMap(partition1, 24000L));
+ .andReturn(singletonMap(partition1, 12000L))
+ .andReturn(singletonMap(partition1, 24000L));
stateManager.checkpoint();
EasyMock.expectLastCall().times(2);
EasyMock.replay(stateManager);
@@ -1640,7 +1564,8 @@ public class StreamTaskTest {
task = createStatefulTask(createConfig(false, "0"), true);
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, singleton(getConsumerRecord(partition1, 10)));
+ task.addRecords(partition1, singleton(getConsumerRecordWithOffsetAsTimestamp(partition1, 10)));
+ task.addRecords(partition2, singleton(getConsumerRecordWithOffsetAsTimestamp(partition2, 10)));
task.process(100L);
assertTrue(task.commitNeeded());
@@ -1651,20 +1576,20 @@ public class StreamTaskTest {
@Test
public void shouldReturnStateManagerChangelogOffsets() {
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.singletonMap(partition1, 50L)).anyTimes();
- EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.singleton(partition1)).anyTimes();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(singletonMap(partition1, 50L)).anyTimes();
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(singleton(partition1)).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
task = createOptimizedStatefulTask(createConfig(false, "100"), consumer);
task.initializeIfNeeded();
- assertEquals(Collections.singletonMap(partition1, 50L), task.changelogOffsets());
+ assertEquals(singletonMap(partition1, 50L), task.changelogOffsets());
task.completeRestoration();
- assertEquals(Collections.singletonMap(partition1, Task.LATEST_OFFSET), task.changelogOffsets());
+ assertEquals(singletonMap(partition1, Task.LATEST_OFFSET), task.changelogOffsets());
}
@Test
@@ -1673,7 +1598,7 @@ public class StreamTaskTest {
EasyMock.expectLastCall().andThrow(new AssertionError("Flush should not be called")).anyTimes();
stateManager.checkpoint();
EasyMock.expectLastCall().andThrow(new AssertionError("Checkpoint should not be called")).anyTimes();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
final MetricName metricName = setupCloseTaskMetric();
@@ -1700,8 +1625,8 @@ public class StreamTaskTest {
stateManager.checkpoint();
EasyMock.expectLastCall().once();
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.emptyMap()).anyTimes();
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(stateManager, recordCollector);
task = createOptimizedStatefulTask(createConfig(false, "100"), consumer);
@@ -1723,20 +1648,20 @@ public class StreamTaskTest {
final long offset = 543L;
final long consumedOffset = 345L;
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.singletonMap(changelogPartition, offset)).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(singletonMap(changelogPartition, offset)).anyTimes();
EasyMock.expectLastCall();
stateManager.checkpoint();
EasyMock.expectLastCall().once();
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(partition1, offset + 12000L));
+ .andReturn(singletonMap(partition1, offset + 12000L));
EasyMock.replay(recordCollector, stateManager);
task = createOptimizedStatefulTask(createConfig(false, "0"), consumer);
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, consumedOffset)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, consumedOffset)));
task.process(100L);
assertTrue(task.commitNeeded());
@@ -1744,7 +1669,7 @@ public class StreamTaskTest {
task.prepareCommit();
task.postCommit(false);
- assertEquals(Task.State.SUSPENDED, task.state());
+ assertEquals(SUSPENDED, task.state());
EasyMock.verify(stateManager);
}
@@ -1756,8 +1681,8 @@ public class StreamTaskTest {
EasyMock.expect(recordCollector.offsets()).andReturn(emptyMap()).anyTimes();
stateManager.checkpoint();
EasyMock.expectLastCall().once();
- EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.singleton(changelogPartition)).anyTimes();
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.singletonMap(changelogPartition, offset)).anyTimes();
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(singleton(changelogPartition)).anyTimes();
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(singletonMap(changelogPartition, offset)).anyTimes();
stateManager.close();
EasyMock.expectLastCall().andThrow(new ProcessorStateException("KABOOM!")).anyTimes();
EasyMock.replay(recordCollector, stateManager);
@@ -1767,7 +1692,7 @@ public class StreamTaskTest {
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, offset)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, offset)));
task.process(100L);
assertTrue(task.commitNeeded());
@@ -1781,7 +1706,7 @@ public class StreamTaskTest {
EasyMock.verify(stateManager);
EasyMock.reset(stateManager);
- EasyMock.expect(stateManager.changelogPartitions()).andStubReturn(Collections.singleton(changelogPartition));
+ EasyMock.expect(stateManager.changelogPartitions()).andStubReturn(singleton(changelogPartition));
stateManager.close();
EasyMock.expectLastCall();
EasyMock.replay(stateManager);
@@ -1791,7 +1716,7 @@ public class StreamTaskTest {
public void shouldThrowOnCloseCleanFlushError() {
final long offset = 543L;
- EasyMock.expect(recordCollector.offsets()).andReturn(Collections.singletonMap(changelogPartition, offset));
+ EasyMock.expect(recordCollector.offsets()).andReturn(singletonMap(changelogPartition, offset));
stateManager.flushCache();
EasyMock.expectLastCall().andThrow(new ProcessorStateException("KABOOM!")).anyTimes();
stateManager.flush();
@@ -1801,7 +1726,7 @@ public class StreamTaskTest {
stateManager.close();
EasyMock.expectLastCall().andThrow(new AssertionError("Close should not be called!")).anyTimes();
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
- EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(emptyMap()).anyTimes();
EasyMock.replay(recordCollector, stateManager);
final MetricName metricName = setupCloseTaskMetric();
@@ -1810,7 +1735,7 @@ public class StreamTaskTest {
task.completeRestoration();
// process one record to make commit needed
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, offset)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, offset)));
task.process(100L);
assertThrows(ProcessorStateException.class, task::prepareCommit);
@@ -1836,14 +1761,14 @@ public class StreamTaskTest {
EasyMock.expectLastCall().andThrow(new AssertionError("Close should not be called!")).anyTimes();
EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet()).anyTimes();
EasyMock.expect(stateManager.changelogOffsets())
- .andReturn(Collections.singletonMap(partition1, offset));
+ .andReturn(singletonMap(partition1, offset));
EasyMock.replay(recordCollector, stateManager);
final MetricName metricName = setupCloseTaskMetric();
task = createOptimizedStatefulTask(createConfig(false, "100"), consumer);
task.initializeIfNeeded();
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, offset)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, offset)));
task.process(100L);
assertTrue(task.commitNeeded());
@@ -1957,12 +1882,12 @@ public class StreamTaskTest {
@Test
public void shouldThrowIfCleanClosingDirtyTask() {
- task = createStatelessTask(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
+ task = createSingleSourceStateless(createConfig(false, "0"), StreamsConfig.METRICS_LATEST);
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, 0)));
- task.process(0L);
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, 0)));
+ assertTrue(task.process(0L));
assertTrue(task.commitNeeded());
assertThrows(TaskMigratedException.class, () -> task.closeClean());
@@ -1974,7 +1899,8 @@ public class StreamTaskTest {
task.initializeIfNeeded();
task.completeRestoration();
- task.addRecords(partition1, singletonList(getConsumerRecord(partition1, 0)));
+ task.addRecords(partition1, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition1, 0)));
+ task.addRecords(partition2, singletonList(getConsumerRecordWithOffsetAsTimestamp(partition2, 0)));
task.process(0L);
assertTrue(task.commitNeeded());
@@ -2140,7 +2066,7 @@ public class StreamTaskTest {
asList(source1, source2),
mkMap(mkEntry(topic1, source1), mkEntry(topic2, source2)),
singletonList(stateStore),
- Collections.emptyMap());
+ emptyMap());
final MockConsumer<byte[], byte[]> consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {
@Override
@@ -2177,7 +2103,7 @@ public class StreamTaskTest {
asList(source1, source3),
mkMap(mkEntry(topic1, source1), mkEntry(topic2, source3)),
singletonList(stateStore),
- Collections.emptyMap());
+ emptyMap());
final InternalProcessorContext context = new ProcessorContextImpl(
taskId,
@@ -2238,6 +2164,44 @@ public class StreamTaskTest {
context);
}
+ private StreamTask createSingleSourceStateless(final StreamsConfig config,
+ final String builtInMetricsVersion) {
+ final ProcessorTopology topology = withSources(
+ asList(source1, processorStreamTime, processorSystemTime),
+ mkMap(mkEntry(topic1, source1))
+ );
+
+ source1.addChild(processorStreamTime);
+ source1.addChild(processorSystemTime);
+
+ EasyMock.expect(stateManager.changelogPartitions()).andReturn(Collections.emptySet());
+ EasyMock.expect(stateManager.changelogOffsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.expect(recordCollector.offsets()).andReturn(Collections.emptyMap()).anyTimes();
+ EasyMock.replay(stateManager, recordCollector);
+
+ final InternalProcessorContext context = new ProcessorContextImpl(
+ taskId,
+ config,
+ stateManager,
+ streamsMetrics,
+ null
+ );
+
+ return new StreamTask(
+ taskId,
+ mkSet(partition1),
+ topology,
+ consumer,
+ config,
+ new StreamsMetricsImpl(metrics, "test", builtInMetricsVersion, time),
+ stateDirectory,
+ cache,
+ time,
+ stateManager,
+ recordCollector,
+ context);
+ }
+
private StreamTask createStatelessTask(final StreamsConfig config,
final String builtInMetricsVersion) {
final ProcessorTopology topology = withSources(
@@ -2315,7 +2279,25 @@ public class StreamTaskTest {
context);
}
- private ConsumerRecord<byte[], byte[]> getConsumerRecord(final TopicPartition topicPartition, final long offset) {
+ private ConsumerRecord<byte[], byte[]> getConsumerRecordWithOffsetAsTimestamp(final TopicPartition topicPartition,
+ final long offset,
+ final int value) {
+ return new ConsumerRecord<>(
+ topicPartition.topic(),
+ topicPartition.partition(),
+ offset,
+ offset, // use the offset as the timestamp
+ TimestampType.CREATE_TIME,
+ 0L,
+ 0,
+ 0,
+ recordKey,
+ intSerializer.serialize(null, value)
+ );
+ }
+
+ private ConsumerRecord<byte[], byte[]> getConsumerRecordWithOffsetAsTimestamp(final TopicPartition topicPartition,
+ final long offset) {
return new ConsumerRecord<>(
topicPartition.topic(),
topicPartition.partition(),
@@ -2330,7 +2312,7 @@ public class StreamTaskTest {
);
}
- private ConsumerRecord<byte[], byte[]> getConsumerRecord(final Integer key, final long offset) {
+ private ConsumerRecord<byte[], byte[]> getConsumerRecordWithOffsetAsTimestamp(final Integer key, final long offset) {
return new ConsumerRecord<>(
topic1,
1,
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 157f9e5..ead343b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -490,7 +490,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
);
thread.setNow(mockTime.milliseconds());
thread.maybeCommit();
@@ -730,7 +731,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
);
thread.setNow(mockTime.milliseconds());
thread.maybeCommit();
@@ -795,7 +797,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
);
thread.setNow(mockTime.milliseconds());
@@ -999,7 +1002,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
thread.setStateListener(
(t, newState, oldState) -> {
@@ -1061,7 +1065,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
final IllegalStateException thrown = assertThrows(
@@ -1101,7 +1106,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
thread.shutdown();
verify(taskManager);
@@ -1134,7 +1140,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
thread.shutdown();
// Execute the run method. Verification of the mock will check that shutdown was only done once
@@ -2041,7 +2048,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
consumer.schedulePollTask(() -> {
@@ -2088,7 +2096,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
consumer.schedulePollTask(() -> {
@@ -2141,7 +2150,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
) {
@Override
void runOnce() {
@@ -2202,7 +2212,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
) {
@Override
void runOnce() {
@@ -2264,7 +2275,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
);
EasyMock.replay(task1, task2, task3, taskManager);
@@ -2433,7 +2445,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
);
assertThat(dummyProducerMetrics, is(thread.producerMetrics()));
@@ -2470,7 +2483,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
HANDLER,
- null
+ null,
+ Integer.MAX_VALUE
);
final MetricName testMetricName = new MetricName("test_metric", "", "", new HashMap<>());
final Metric testMetric = new KafkaMetric(
@@ -2519,7 +2533,8 @@ public class StreamThreadTest {
new AtomicLong(Long.MAX_VALUE),
null,
e -> { },
- null
+ null,
+ Integer.MAX_VALUE
) {
@Override
void runOnce() {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
index 93ce84b..5e00900 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
@@ -24,6 +24,7 @@ import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Metric;
@@ -3002,6 +3003,11 @@ public class TaskManagerTest {
}
@Override
+ public void addFetchedMetadata(final TopicPartition partition, final ConsumerRecords.Metadata metadata) {
+ // do nothing
+ }
+
+ @Override
public boolean process(final long wallClockTime) {
if (isActive() && state() == State.RUNNING) {
for (final LinkedList<ConsumerRecord<byte[], byte[]>> records : queue.values()) {
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/LogCaptureAppender.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/LogCaptureAppender.java
index 296356f..41d15da 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/LogCaptureAppender.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/LogCaptureAppender.java
@@ -69,6 +69,10 @@ public class LogCaptureAppender extends AppenderSkeleton implements AutoCloseabl
Logger.getLogger(clazz).setLevel(Level.DEBUG);
}
+ public static void setClassLoggerToTrace(final Class<?> clazz) {
+ Logger.getLogger(clazz).setLevel(Level.TRACE);
+ }
+
public static void unregister(final LogCaptureAppender logCaptureAppender) {
Logger.getRootLogger().removeAppender(logCaptureAppender);
}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 8a90076..68f4a6b 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -19,6 +19,7 @@ package org.apache.kafka.streams;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.MockConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
@@ -228,6 +229,7 @@ public class TopologyTestDriver implements Closeable {
private final Map<String, TopicPartition> partitionsByInputTopic = new HashMap<>();
private final Map<String, TopicPartition> globalPartitionsByInputTopic = new HashMap<>();
private final Map<TopicPartition, AtomicLong> offsetsByTopicOrPatternPartition = new HashMap<>();
+ private final Map<TopicPartition, ConsumerRecords.Metadata> metadataByPartition = new HashMap<>();
private final Map<String, Queue<ProducerRecord<byte[], byte[]>>> outputRecordsByTopic = new HashMap<>();
private final StreamThread.ProcessingMode processingMode;
@@ -424,6 +426,10 @@ public class TopologyTestDriver implements Closeable {
final TopicPartition tp = new TopicPartition(topic, PARTITION_ID);
partitionsByInputTopic.put(topic, tp);
offsetsByTopicOrPatternPartition.put(tp, new AtomicLong());
+ metadataByPartition.put(
+ tp,
+ new ConsumerRecords.Metadata(mockWallClockTime.milliseconds(), 0L, 0L, 0L)
+ );
}
final boolean createStateDirectory = processorTopology.hasPersistentLocalStore() ||
@@ -591,10 +597,11 @@ public class TopologyTestDriver implements Closeable {
final byte[] key,
final byte[] value,
final Headers headers) {
+ final long offset = offsetsByTopicOrPatternPartition.get(topicOrPatternPartition).incrementAndGet() - 1;
task.addRecords(topicOrPatternPartition, Collections.singleton(new ConsumerRecord<>(
inputTopic,
topicOrPatternPartition.partition(),
- offsetsByTopicOrPatternPartition.get(topicOrPatternPartition).incrementAndGet() - 1,
+ offset,
timestamp,
TimestampType.CREATE_TIME,
(long) ConsumerRecord.NULL_CHECKSUM,
@@ -604,6 +611,13 @@ public class TopologyTestDriver implements Closeable {
value,
headers))
);
+ final ConsumerRecords.Metadata metadata = new ConsumerRecords.Metadata(
+ mockWallClockTime.milliseconds(),
+ offset,
+ 0L,
+ offset
+ );
+ metadataByPartition.put(topicOrPatternPartition, metadata);
}
private void completeAllProcessableWork() {
@@ -618,6 +632,7 @@ public class TopologyTestDriver implements Closeable {
// For this method, it just means there's nothing to do.
if (task != null) {
while (task.hasRecordsQueued() && task.isProcessable(mockWallClockTime.milliseconds())) {
+ refreshTaskMetadata(); // isProcessable consumes the metadata, so we need to refresh again
// Process the record ...
task.process(mockWallClockTime.milliseconds());
task.maybePunctuateStreamTime();
@@ -725,6 +740,21 @@ public class TopologyTestDriver implements Closeable {
);
}
}
+
+ refreshTaskMetadata();
+ }
+
+ /**
+ * We're not modelling any async poll dynamics, so we call this method before each
+ * {@link StreamTask#isProcessable(long)} and {@link StreamTask#process(long)} so
+ * that the task will have full metadata for each partition. In particular, we want
+ * it to know that its lag is zero on all inputs, so it doesn't think it needs to wait
+ * for more polls.
+ */
+ private void refreshTaskMetadata() {
+ for (final Map.Entry<TopicPartition, ConsumerRecords.Metadata> entry : metadataByPartition.entrySet()) {
+ task.addFetchedMetadata(entry.getKey(), entry.getValue());
+ }
}
/**