You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jo...@apache.org on 2025/04/30 14:28:13 UTC
(beam) branch master updated: [KafkaIO] Improve caching in backlog estimation and processing (#34331)
This is an automated email from the ASF dual-hosted git repository.
johncasey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9628c1ef9ba [KafkaIO] Improve caching in backlog estimation and processing (#34331)
9628c1ef9ba is described below
commit 9628c1ef9ba95e5a79b12208e914b1f96703a752
Author: Steven van Rossum <sj...@google.com>
AuthorDate: Wed Apr 30 16:28:07 2025 +0200
[KafkaIO] Improve caching in backlog estimation and processing (#34331)
* Improve caching in backlog estimation and processing
* Add comment to explain the behavior of volatile guard field in KafkaLatestOffsetEstimator
* Guard against exceptions in endOffset refresh
* Call cancelIfTimeouted in roundtripElements to shutdown lingering pipelines
* Add missing calls to seek and/or pause before return points added in #34202
---
.../org/apache/beam/sdk/io/kafka/KafkaIOUtils.java | 15 +
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 720 ++++++++++++---------
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 7 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 2 +
4 files changed, 422 insertions(+), 322 deletions(-)
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java
index 8b778ce5481..f404c8d9640 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.kafka;
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
@@ -130,6 +131,20 @@ public final class KafkaIOUtils {
return offsetConsumerConfig;
}
+ static Map<String, Object> overrideBootstrapServersConfig(
+ Map<String, Object> currentConfig, KafkaSourceDescriptor description) {
+ checkState(
+ currentConfig.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)
+ || description.getBootStrapServers() != null);
+ Map<String, Object> config = new HashMap<>(currentConfig);
+ if (description.getBootStrapServers() != null && description.getBootStrapServers().size() > 0) {
+ config.put(
+ ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
+ String.join(",", description.getBootStrapServers()));
+ }
+ return config;
+ }
+
/*
* Maintains approximate average over last 1000 elements.
* Usage is only thread-safe for a single producer and multiple consumers.
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java
index b2424875d7f..cd3ad2526d3 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java
@@ -19,18 +19,19 @@ package org.apache.beam.sdk.io.kafka;
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState;
+import java.io.Closeable;
import java.math.BigDecimal;
import java.math.MathContext;
import java.time.Duration;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.kafka.KafkaIO.ReadSourceDescriptors;
import org.apache.beam.sdk.io.kafka.KafkaIOUtils.MovingAvg;
@@ -51,19 +52,22 @@ import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.HasProgr
import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator;
import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators.MonotonicallyIncreasing;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.MemoizingPerInstantiationSerializableSupplier;
import org.apache.beam.sdk.util.Preconditions;
+import org.apache.beam.sdk.util.SerializableSupplier;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalCause;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closeables;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -71,6 +75,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.serialization.Deserializer;
import org.checkerframework.checker.nullness.qual.Nullable;
@@ -187,19 +192,98 @@ abstract class ReadFromKafkaDoFn<K, V>
private ReadFromKafkaDoFn(
ReadSourceDescriptors<K, V> transform,
TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag) {
+ final SerializableFunction<Map<String, Object>, Consumer<byte[], byte[]>> consumerFactoryFn =
+ transform.getConsumerFactoryFn();
this.consumerConfig = transform.getConsumerConfig();
- this.offsetConsumerConfig = transform.getOffsetConsumerConfig();
this.keyDeserializerProvider =
Preconditions.checkArgumentNotNull(transform.getKeyDeserializerProvider());
this.valueDeserializerProvider =
Preconditions.checkArgumentNotNull(transform.getValueDeserializerProvider());
- this.consumerFactoryFn = transform.getConsumerFactoryFn();
this.extractOutputTimestampFn = transform.getExtractOutputTimestampFn();
this.createWatermarkEstimatorFn = transform.getCreateWatermarkEstimatorFn();
this.timestampPolicyFactory = transform.getTimestampPolicyFactory();
this.checkStopReadingFn = transform.getCheckStopReadingFn();
this.badRecordRouter = transform.getBadRecordRouter();
this.recordTag = recordTag;
+ this.avgRecordSizeCacheSupplier =
+ new MemoizingPerInstantiationSerializableSupplier<>(
+ () ->
+ CacheBuilder.newBuilder()
+ .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .weakValues()
+ .build(
+ new CacheLoader<KafkaSourceDescriptor, MovingAvg>() {
+ @Override
+ public MovingAvg load(KafkaSourceDescriptor kafkaSourceDescriptor)
+ throws Exception {
+ return new MovingAvg();
+ }
+ }));
+ this.latestOffsetEstimatorCacheSupplier =
+ new MemoizingPerInstantiationSerializableSupplier<>(
+ () ->
+ CacheBuilder.newBuilder()
+ .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .weakValues()
+ .removalListener(
+ (RemovalNotification<KafkaSourceDescriptor, KafkaLatestOffsetEstimator>
+ notification) -> {
+ final @Nullable KafkaLatestOffsetEstimator value;
+ if (notification.getCause() == RemovalCause.COLLECTED
+ && (value = notification.getValue()) != null) {
+ value.close();
+ }
+ })
+ .build(
+ new CacheLoader<KafkaSourceDescriptor, KafkaLatestOffsetEstimator>() {
+ @Override
+ public KafkaLatestOffsetEstimator load(
+ final KafkaSourceDescriptor sourceDescriptor) {
+ LOG.info(
+ "Creating Kafka consumer for offset estimation for {}",
+ sourceDescriptor);
+ final Map<String, Object> config =
+ KafkaIOUtils.overrideBootstrapServersConfig(
+ consumerConfig, sourceDescriptor);
+ final Consumer<byte[], byte[]> consumer =
+ consumerFactoryFn.apply(config);
+ return new KafkaLatestOffsetEstimator(
+ consumer, sourceDescriptor.getTopicPartition());
+ }
+ }));
+ this.pollConsumerCacheSupplier =
+ new MemoizingPerInstantiationSerializableSupplier<>(
+ () ->
+ CacheBuilder.newBuilder()
+ .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .weakValues()
+ .removalListener(
+ (RemovalNotification<KafkaSourceDescriptor, Consumer<byte[], byte[]>>
+ notification) -> {
+ final @Nullable Consumer<byte[], byte[]> value;
+ if (notification.getCause() == RemovalCause.COLLECTED
+ && (value = notification.getValue()) != null) {
+ value.close();
+ }
+ })
+ .build(
+ new CacheLoader<KafkaSourceDescriptor, Consumer<byte[], byte[]>>() {
+ @Override
+ public Consumer<byte[], byte[]> load(
+ KafkaSourceDescriptor sourceDescriptor) {
+ LOG.info(
+ "Creating Kafka consumer for restriction processing for {}",
+ sourceDescriptor);
+ final Map<String, Object> config =
+ KafkaIOUtils.overrideBootstrapServersConfig(
+ consumerConfig, sourceDescriptor);
+ final Consumer<byte[], byte[]> consumer =
+ consumerFactoryFn.apply(config);
+ consumer.assign(
+ Collections.singleton(sourceDescriptor.getTopicPartition()));
+ return consumer;
+ }
+ }));
this.consumerPollingTimeout =
Duration.ofSeconds(
transform.getConsumerPollingTimeout() > 0
@@ -209,29 +293,10 @@ abstract class ReadFromKafkaDoFn<K, V>
private static final Logger LOG = LoggerFactory.getLogger(ReadFromKafkaDoFn.class);
- /**
- * A holder class for all construction time unique instances of {@link ReadFromKafkaDoFn}. Caches
- * must run clean up tasks when {@link #teardown()} is called.
- */
- private static final class SharedStateHolder {
-
- private static final Map<Long, LoadingCache<KafkaSourceDescriptor, KafkaLatestOffsetEstimator>>
- OFFSET_ESTIMATOR_CACHE = new ConcurrentHashMap<>();
- private static final Map<Long, LoadingCache<KafkaSourceDescriptor, MovingAvg>>
- AVG_RECORD_SIZE_CACHE = new ConcurrentHashMap<>();
- }
-
- private static final AtomicLong FN_ID = new AtomicLong();
-
- // A unique identifier for the instance. Generally unique unless the ID generator overflows.
- private final long fnId = FN_ID.getAndIncrement();
-
- private final @Nullable Map<String, Object> offsetConsumerConfig;
+ private static final Joiner COMMA_JOINER = Joiner.on(',');
private final @Nullable CheckStopReadingFn checkStopReadingFn;
- private final SerializableFunction<Map<String, Object>, Consumer<byte[], byte[]>>
- consumerFactoryFn;
private final @Nullable SerializableFunction<KafkaRecord<K, V>, Instant> extractOutputTimestampFn;
private final @Nullable SerializableFunction<Instant, WatermarkEstimator<Instant>>
createWatermarkEstimatorFn;
@@ -241,13 +306,19 @@ abstract class ReadFromKafkaDoFn<K, V>
private final TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag;
+ private final SerializableSupplier<LoadingCache<KafkaSourceDescriptor, MovingAvg>>
+ avgRecordSizeCacheSupplier;
+
+ private final SerializableSupplier<
+ LoadingCache<KafkaSourceDescriptor, KafkaLatestOffsetEstimator>>
+ latestOffsetEstimatorCacheSupplier;
+
+ private final SerializableSupplier<LoadingCache<KafkaSourceDescriptor, Consumer<byte[], byte[]>>>
+ pollConsumerCacheSupplier;
+
// Valid between bundle start and bundle finish.
private transient @Nullable Deserializer<K> keyDeserializerInstance = null;
private transient @Nullable Deserializer<V> valueDeserializerInstance = null;
- private transient @Nullable LoadingCache<KafkaSourceDescriptor, KafkaLatestOffsetEstimator>
- offsetEstimatorCache;
-
- private transient @Nullable LoadingCache<KafkaSourceDescriptor, MovingAvg> avgRecordSizeCache;
private static final long DEFAULT_KAFKA_POLL_TIMEOUT = 2L;
@VisibleForTesting final Duration consumerPollingTimeout;
@VisibleForTesting final DeserializerProvider<K> keyDeserializerProvider;
@@ -263,82 +334,169 @@ abstract class ReadFromKafkaDoFn<K, V>
* fetch backlog.
*/
private static class KafkaLatestOffsetEstimator
- implements GrowableOffsetRangeTracker.RangeEndEstimator {
-
+ implements GrowableOffsetRangeTracker.RangeEndEstimator, Closeable {
+ private static final AtomicReferenceFieldUpdater<KafkaLatestOffsetEstimator, @Nullable Runnable>
+ CURRENT_REFRESH_TASK =
+ (AtomicReferenceFieldUpdater<KafkaLatestOffsetEstimator, @Nullable Runnable>)
+ AtomicReferenceFieldUpdater.newUpdater(
+ KafkaLatestOffsetEstimator.class, Runnable.class, "currentRefreshTask");
+ private final Executor executor;
private final Consumer<byte[], byte[]> offsetConsumer;
private final TopicPartition topicPartition;
- private final Supplier<Long> memoizedBacklog;
+ // TODO(sjvanrossum): Use VarHandle.setOpaque/getOpaque when Java 8 support is dropped
+ private long lastRefreshEndOffset;
+ // TODO(sjvanrossum): Use VarHandle.setOpaque/getOpaque when Java 8 support is dropped
+ private long nextRefreshNanos;
+ private volatile @Nullable Runnable currentRefreshTask;
+
+ /*
+ Periodic refreshes of lastRefreshEndOffset and nextRefreshNanos are guarded by the volatile
+ field currentRefreshTask. This guard's correctness depends on specific ordering of reads and
+ writes (loads and stores).
+
+ To validate the behavior of this guard please read the Java Memory Model (JMM) specification.
+ For the current context consider the following oversimplifications of the JMM:
+ - Writes to a non-volatile long or double field are non-atomic.
+ - Writes to a non-volatile field may never become visible to another core.
+ - Writes to a volatile field are atomic and will become visible to another core.
+ - Lazy writes to a volatile field are atomic and will become visible to another core for
+ reads of that volatile field.
+ - Writes preceeding writes or lazy writes to a volatile field are visible to another core.
+
+ In short, the contents of this class' guarded fields are visible if the guard field is (lazily)
+ written last and read first. The contents of the volatile guard may be stale in comparison to
+ the contents of the guarded fields. For this method it is important that no more than one
+ thread will schedule a refresh task. Using currentRefreshTask as the guard field ensures that
+ lastRefreshEndOffset and nextRefreshNanos are at least as stale as currentRefreshTask.
+ It's fine if lastRefreshEndOffset and nextRefreshNanos are less stale than currentRefreshTask.
+
+ Removing currentRefreshTask by guarding on nextRefreshNanos is possible, but executing
+ currentRefreshTask == null is practically free (measured in cycles) compared to executing
+ nextRefreshNanos < System.nanoTime() (measured in nanoseconds).
+
+ Note that the JMM specifies that writes to a long or double are not guaranteed to be atomic.
+ In practice, every 64-bit JVM will treat them as atomic (and the JMM encourages this).
+ There's no way to force atomicity without visibility in Java 8 so atomicity guards have been
+ omitted. Java 9 introduces VarHandle with "opaque" getters/setters which do provide this.
+ */
KafkaLatestOffsetEstimator(
- Consumer<byte[], byte[]> offsetConsumer, TopicPartition topicPartition) {
+ final Consumer<byte[], byte[]> offsetConsumer, final TopicPartition topicPartition) {
+ this.executor = Executors.newSingleThreadExecutor();
this.offsetConsumer = offsetConsumer;
this.topicPartition = topicPartition;
- memoizedBacklog =
- Suppliers.memoizeWithExpiration(
- () -> {
- synchronized (offsetConsumer) {
- return Preconditions.checkStateNotNull(
- offsetConsumer
- .endOffsets(Collections.singleton(topicPartition))
- .get(topicPartition),
- "No end offset found for partition %s.",
- topicPartition);
- }
- },
- 1,
- TimeUnit.SECONDS);
+ this.lastRefreshEndOffset = -1L;
+ this.nextRefreshNanos = Long.MIN_VALUE;
+ this.currentRefreshTask = null;
}
@Override
- protected void finalize() {
- try {
- Closeables.close(offsetConsumer, true);
- LOG.info("Offset Estimator consumer was closed for {}", topicPartition);
- } catch (Exception anyException) {
- LOG.warn("Failed to close offset consumer for {}", topicPartition);
+ public long estimate() {
+ final @Nullable Runnable task = currentRefreshTask; // volatile load (acquire)
+
+ final long currentNanos;
+ if (task == null
+ && nextRefreshNanos < (currentNanos = System.nanoTime()) // normal load
+ && CURRENT_REFRESH_TASK.compareAndSet(this, null, this::refresh)) { // volatile load/store
+ try {
+ executor.execute(this::refresh);
+ } catch (RejectedExecutionException ex) {
+ LOG.error("Execution of end offset refresh rejected for {}", topicPartition, ex);
+ nextRefreshNanos = currentNanos + TimeUnit.SECONDS.toNanos(1); // normal store
+ CURRENT_REFRESH_TASK.lazySet(this, null); // ordered store (release)
+ }
}
+
+ return lastRefreshEndOffset; // normal load
}
@Override
- public long estimate() {
- return memoizedBacklog.get();
+ public void close() {
+ offsetConsumer.close();
+ }
+
+ private void refresh() {
+ try {
+ @Nullable
+ Long endOffset =
+ offsetConsumer.endOffsets(Collections.singleton(topicPartition)).get(topicPartition);
+ if (endOffset == null) {
+ LOG.warn("No end offset found for partition {}.", topicPartition);
+ } else {
+ lastRefreshEndOffset = endOffset; // normal store
+ }
+ nextRefreshNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); // normal store
+ } finally {
+ CURRENT_REFRESH_TASK.lazySet(this, null); // ordered store (release)
+ }
}
}
@GetInitialRestriction
public OffsetRange initialRestriction(@Element KafkaSourceDescriptor kafkaSourceDescriptor) {
- Map<String, Object> updatedConsumerConfig =
- overrideBootstrapServersConfig(consumerConfig, kafkaSourceDescriptor);
- TopicPartition partition = kafkaSourceDescriptor.getTopicPartition();
- LOG.info("Creating Kafka consumer for initial restriction for {}", kafkaSourceDescriptor);
- try (Consumer<byte[], byte[]> offsetConsumer = consumerFactoryFn.apply(updatedConsumerConfig)) {
- offsetConsumer.assign(ImmutableList.of(partition));
- long startOffset;
- @Nullable Instant startReadTime = kafkaSourceDescriptor.getStartReadTime();
- if (kafkaSourceDescriptor.getStartReadOffset() != null) {
- startOffset = kafkaSourceDescriptor.getStartReadOffset();
- } else if (startReadTime != null) {
- startOffset = ConsumerSpEL.offsetForTime(offsetConsumer, partition, startReadTime);
- } else {
- startOffset = offsetConsumer.position(partition);
- }
+ final Consumer<byte[], byte[]> consumer =
+ pollConsumerCacheSupplier.get().getUnchecked(kafkaSourceDescriptor);
+
+ final long startOffset;
+ final long stopOffset;
+
+ final @Nullable Long startReadOffset = kafkaSourceDescriptor.getStartReadOffset();
+ final @Nullable Instant startReadTime = kafkaSourceDescriptor.getStartReadTime();
+ if (startReadOffset != null) {
+ startOffset = startReadOffset;
+ } else if (startReadTime != null) {
+ startOffset =
+ Preconditions.checkStateNotNull(
+ consumer
+ .offsetsForTimes(
+ Collections.singletonMap(
+ kafkaSourceDescriptor.getTopicPartition(), startReadTime.getMillis()))
+ .get(kafkaSourceDescriptor.getTopicPartition()))
+ .offset();
+ } else {
+ startOffset = consumer.position(kafkaSourceDescriptor.getTopicPartition());
+ }
- long endOffset = Long.MAX_VALUE;
- @Nullable Instant stopReadTime = kafkaSourceDescriptor.getStopReadTime();
- if (kafkaSourceDescriptor.getStopReadOffset() != null) {
- endOffset = kafkaSourceDescriptor.getStopReadOffset();
- } else if (stopReadTime != null) {
- endOffset = ConsumerSpEL.offsetForTime(offsetConsumer, partition, stopReadTime);
- }
- new OffsetRange(startOffset, endOffset);
- Lineage.getSources()
- .add(
- "kafka",
- ImmutableList.of(
- (String) updatedConsumerConfig.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG),
- MoreObjects.firstNonNull(kafkaSourceDescriptor.getTopic(), partition.topic())));
- return new OffsetRange(startOffset, endOffset);
+ final @Nullable Long stopReadOffset = kafkaSourceDescriptor.getStopReadOffset();
+ final @Nullable Instant stopReadTime = kafkaSourceDescriptor.getStopReadTime();
+ if (stopReadOffset != null) {
+ stopOffset = stopReadOffset;
+ } else if (stopReadTime != null) {
+ stopOffset =
+ Preconditions.checkStateNotNull(
+ consumer
+ .offsetsForTimes(
+ Collections.singletonMap(
+ kafkaSourceDescriptor.getTopicPartition(), stopReadTime.getMillis()))
+ .get(kafkaSourceDescriptor.getTopicPartition()))
+ .offset();
+ } else {
+ stopOffset = Long.MAX_VALUE;
}
+
+ final OffsetRange initialRestriction = new OffsetRange(startOffset, stopOffset);
+ Lineage.getSources()
+ .add(
+ "kafka",
+ ImmutableList.of(
+ Optional.ofNullable(
+ KafkaIOUtils.overrideBootstrapServersConfig(
+ consumerConfig, kafkaSourceDescriptor)
+ .get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG))
+ .map(
+ value ->
+ (@Nullable List<String>)
+ ConfigDef.parseType(
+ ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
+ value,
+ ConfigDef.Type.LIST))
+ .map(ImmutableSet::copyOf)
+ .map(COMMA_JOINER::join)
+ .get(),
+ MoreObjects.firstNonNull(
+ kafkaSourceDescriptor.getTopic(),
+ kafkaSourceDescriptor.getTopicPartition().topic())));
+ return initialRestriction;
}
@GetInitialWatermarkEstimatorState
@@ -356,13 +514,11 @@ abstract class ReadFromKafkaDoFn<K, V>
@GetSize
public double getSize(
- @Element KafkaSourceDescriptor kafkaSourceDescriptor, @Restriction OffsetRange offsetRange)
- throws ExecutionException {
- // If present, estimates the record size.
- final LoadingCache<KafkaSourceDescriptor, MovingAvg> avgRecordSizeCache =
- Preconditions.checkStateNotNull(this.avgRecordSizeCache);
+ @Element KafkaSourceDescriptor kafkaSourceDescriptor, @Restriction OffsetRange offsetRange) {
+ // If present, estimates the record size to offset gap ratio. Compacted topics may hold less
+ // records than the estimated offset range due to record deletion within a partition.
final @Nullable MovingAvg avgRecordSize =
- avgRecordSizeCache.getIfPresent(kafkaSourceDescriptor);
+ avgRecordSizeCacheSupplier.get().getIfPresent(kafkaSourceDescriptor);
// The tracker estimates the offset range by subtracting the last claimed position from the
// currently observed end offset for the partition belonging to this split.
final double estimatedOffsetRange =
@@ -378,8 +534,7 @@ abstract class ReadFromKafkaDoFn<K, V>
@NewTracker
public OffsetRangeTracker restrictionTracker(
- @Element KafkaSourceDescriptor kafkaSourceDescriptor, @Restriction OffsetRange restriction)
- throws ExecutionException {
+ @Element KafkaSourceDescriptor kafkaSourceDescriptor, @Restriction OffsetRange restriction) {
if (restriction.getTo() < Long.MAX_VALUE) {
return new OffsetRangeTracker(restriction);
}
@@ -387,12 +542,9 @@ abstract class ReadFromKafkaDoFn<K, V>
// OffsetEstimators are cached for each topic-partition because they hold a stateful connection,
// so we want to minimize the amount of connections that we start and track with Kafka. Another
// point is that it has a memoized backlog, and this should make that more reusable estimations.
- final LoadingCache<KafkaSourceDescriptor, KafkaLatestOffsetEstimator> offsetEstimatorCache =
- Preconditions.checkStateNotNull(this.offsetEstimatorCache);
- final KafkaLatestOffsetEstimator offsetEstimator =
- offsetEstimatorCache.get(kafkaSourceDescriptor);
-
- return new GrowableOffsetRangeTracker(restriction.getFrom(), offsetEstimator);
+ return new GrowableOffsetRangeTracker(
+ restriction.getFrom(),
+ latestOffsetEstimatorCacheSupplier.get().getUnchecked(kafkaSourceDescriptor));
}
@ProcessElement
@@ -402,10 +554,11 @@ abstract class ReadFromKafkaDoFn<K, V>
WatermarkEstimator<Instant> watermarkEstimator,
MultiOutputReceiver receiver)
throws Exception {
- final LoadingCache<KafkaSourceDescriptor, MovingAvg> avgRecordSizeCache =
- Preconditions.checkStateNotNull(this.avgRecordSizeCache);
- final LoadingCache<KafkaSourceDescriptor, KafkaLatestOffsetEstimator> offsetEstimatorCache =
- Preconditions.checkStateNotNull(this.offsetEstimatorCache);
+ final MovingAvg avgRecordSize = avgRecordSizeCacheSupplier.get().get(kafkaSourceDescriptor);
+ final KafkaLatestOffsetEstimator latestOffsetEstimator =
+ latestOffsetEstimatorCacheSupplier.get().get(kafkaSourceDescriptor);
+ final Consumer<byte[], byte[]> consumer =
+ pollConsumerCacheSupplier.get().get(kafkaSourceDescriptor);
final Deserializer<K> keyDeserializerInstance =
Preconditions.checkStateNotNull(this.keyDeserializerInstance);
final Deserializer<V> valueDeserializerInstance =
@@ -420,15 +573,12 @@ abstract class ReadFromKafkaDoFn<K, V>
METRIC_NAMESPACE, RAW_SIZE_METRIC_PREFIX + "backlogBytes_" + topicPartition.toString());
// Stop processing current TopicPartition when it's time to stop.
- if (checkStopReadingFn != null
- && checkStopReadingFn.apply(kafkaSourceDescriptor.getTopicPartition())) {
+ if (checkStopReadingFn != null && checkStopReadingFn.apply(topicPartition)) {
// Attempt to claim the last element in the restriction, such that the restriction tracker
// doesn't throw an exception when checkDone is called
tracker.tryClaim(tracker.currentRestriction().getTo() - 1);
return ProcessContinuation.stop();
}
- Map<String, Object> updatedConsumerConfig =
- overrideBootstrapServersConfig(consumerConfig, kafkaSourceDescriptor);
// If there is a timestampPolicyFactory, create the TimestampPolicy for current
// TopicPartition.
TimestampPolicy<K, V> timestampPolicy = null;
@@ -438,159 +588,156 @@ abstract class ReadFromKafkaDoFn<K, V>
topicPartition, Optional.ofNullable(watermarkEstimator.currentWatermark()));
}
- LOG.info("Creating Kafka consumer for process continuation for {}", kafkaSourceDescriptor);
- try (Consumer<byte[], byte[]> consumer = consumerFactoryFn.apply(updatedConsumerConfig)) {
- consumer.assign(ImmutableList.of(kafkaSourceDescriptor.getTopicPartition()));
- long startOffset = tracker.currentRestriction().getFrom();
- long expectedOffset = startOffset;
- consumer.seek(kafkaSourceDescriptor.getTopicPartition(), startOffset);
- long skippedRecords = 0L;
- final Stopwatch sw = Stopwatch.createStarted();
+ long startOffset = tracker.currentRestriction().getFrom();
+ long expectedOffset = startOffset;
+ consumer.resume(Collections.singleton(topicPartition));
+ consumer.seek(topicPartition, startOffset);
+ long skippedRecords = 0L;
+ final Stopwatch sw = Stopwatch.createStarted();
- final KafkaMetrics kafkaMetrics = KafkaSinkMetrics.kafkaMetrics();
- try {
- while (true) {
- // Fetch the record size accumulator.
- final MovingAvg avgRecordSize = avgRecordSizeCache.getUnchecked(kafkaSourceDescriptor);
- // TODO: Remove this timer and use the existing fetch-latency-avg metric.
- // A consumer will often have prefetches waiting to be returned immediately in which case
- // this timer may contribute more latency than it measures.
- // See https://shipilev.net/blog/2014/nanotrusting-nanotime/ for more information.
- final Stopwatch pollTimer = Stopwatch.createStarted();
- // Fetch the next records.
- final ConsumerRecords<byte[], byte[]> rawRecords =
- consumer.poll(this.consumerPollingTimeout);
- kafkaMetrics.updateSuccessfulRpcMetrics(topicPartition.topic(), pollTimer.elapsed());
-
- // No progress when the polling timeout expired.
- // Self-checkpoint and move to process the next element.
- if (rawRecords == ConsumerRecords.<byte[], byte[]>empty()) {
- if (!topicPartitionExists(
- kafkaSourceDescriptor.getTopicPartition(),
- consumer.partitionsFor(kafkaSourceDescriptor.getTopic()))) {
- return ProcessContinuation.stop();
- }
- if (timestampPolicy != null) {
- updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
- }
- return ProcessContinuation.resume();
+ final KafkaMetrics kafkaMetrics = KafkaSinkMetrics.kafkaMetrics();
+ try {
+ while (true) {
+ // TODO: Remove this timer and use the existing fetch-latency-avg metric.
+ // A consumer will often have prefetches waiting to be returned immediately in which case
+ // this timer may contribute more latency than it measures.
+ // See https://shipilev.net/blog/2014/nanotrusting-nanotime/ for more information.
+ final Stopwatch pollTimer = Stopwatch.createStarted();
+ // Fetch the next records.
+ final ConsumerRecords<byte[], byte[]> rawRecords =
+ consumer.poll(this.consumerPollingTimeout);
+ kafkaMetrics.updateSuccessfulRpcMetrics(topicPartition.topic(), pollTimer.elapsed());
+
+ // No progress when the polling timeout expired.
+ // Self-checkpoint and move to process the next element.
+ if (rawRecords == ConsumerRecords.<byte[], byte[]>empty()) {
+ consumer.pause(Collections.singleton(topicPartition));
+
+ if (!topicPartitionExists(
+ kafkaSourceDescriptor.getTopicPartition(),
+ consumer.partitionsFor(kafkaSourceDescriptor.getTopic()))) {
+ return ProcessContinuation.stop();
+ }
+ if (timestampPolicy != null) {
+ updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
}
+ return ProcessContinuation.resume();
+ }
- // Visible progress within the consumer polling timeout.
- // Partially or fully claim and process records in this batch.
- for (ConsumerRecord<byte[], byte[]> rawRecord : rawRecords) {
- // If the Kafka consumer returns a record with an offset that is already processed
- // the record can be safely skipped. This is needed because there is a possibility
- // that the seek() above fails to move the offset to the desired position. In which
- // case poll() would return records that are already cnsumed.
- if (rawRecord.offset() < startOffset) {
- // If the start offset is not reached even after skipping the records for 10 seconds
- // then the processing is stopped with a backoff to give the Kakfa server some time
- // catch up.
- if (sw.elapsed().getSeconds() > 10L) {
- LOG.error(
- "The expected offset ({}) was not reached even after"
- + " skipping consumed records for 10 seconds. The offset we could"
- + " reach was {}. The processing of this bundle will be attempted"
- + " at a later time.",
- expectedOffset,
- rawRecord.offset());
- return ProcessContinuation.resume()
- .withResumeDelay(org.joda.time.Duration.standardSeconds(10L));
- }
- skippedRecords++;
- continue;
- }
- if (skippedRecords > 0L) {
- LOG.warn(
- "{} records were skipped due to seek returning an"
- + " earlier position than requested position of {}",
- skippedRecords,
- expectedOffset);
- skippedRecords = 0L;
- }
- if (!tracker.tryClaim(rawRecord.offset())) {
- return ProcessContinuation.stop();
- }
- expectedOffset = rawRecord.offset() + 1;
- try {
- KafkaRecord<K, V> kafkaRecord =
- new KafkaRecord<>(
- rawRecord.topic(),
- rawRecord.partition(),
- rawRecord.offset(),
- ConsumerSpEL.getRecordTimestamp(rawRecord),
- ConsumerSpEL.getRecordTimestampType(rawRecord),
- ConsumerSpEL.hasHeaders() ? rawRecord.headers() : null,
- ConsumerSpEL.deserializeKey(keyDeserializerInstance, rawRecord),
- ConsumerSpEL.deserializeValue(valueDeserializerInstance, rawRecord));
- int recordSize =
- (rawRecord.key() == null ? 0 : rawRecord.key().length)
- + (rawRecord.value() == null ? 0 : rawRecord.value().length);
- avgRecordSize.update(recordSize);
- rawSizes.update(recordSize);
- Instant outputTimestamp;
- // The outputTimestamp and watermark will be computed by timestampPolicy, where the
- // WatermarkEstimator should be a manual one.
- if (timestampPolicy != null) {
- TimestampPolicyContext context =
- updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
- outputTimestamp = timestampPolicy.getTimestampForRecord(context, kafkaRecord);
- } else {
- Preconditions.checkStateNotNull(this.extractOutputTimestampFn);
- outputTimestamp = extractOutputTimestampFn.apply(kafkaRecord);
- }
- receiver
- .get(recordTag)
- .outputWithTimestamp(KV.of(kafkaSourceDescriptor, kafkaRecord), outputTimestamp);
- } catch (SerializationException e) {
- // This exception should only occur during the key and value deserialization when
- // creating the Kafka Record
- badRecordRouter.route(
- receiver,
- rawRecord,
- null,
- e,
- "Failure deserializing Key or Value of Kakfa record reading from Kafka");
- if (timestampPolicy != null) {
- updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
- }
+ // Visible progress within the consumer polling timeout.
+ // Partially or fully claim and process records in this batch.
+ for (ConsumerRecord<byte[], byte[]> rawRecord : rawRecords) {
+ // If the Kafka consumer returns a record with an offset that is already processed
+ // the record can be safely skipped. This is needed because there is a possibility
+ // that the seek() above fails to move the offset to the desired position. In which
+ // case poll() would return records that are already cnsumed.
+ if (rawRecord.offset() < startOffset) {
+ // If the start offset is not reached even after skipping the records for 10 seconds
+ // then the processing is stopped with a backoff to give the Kakfa server some time
+ // catch up.
+ if (sw.elapsed().getSeconds() > 10L) {
+ LOG.error(
+ "The expected offset ({}) was not reached even after"
+ + " skipping consumed records for 10 seconds. The offset we could"
+ + " reach was {}. The processing of this bundle will be attempted"
+ + " at a later time.",
+ expectedOffset,
+ rawRecord.offset());
+ consumer.pause(Collections.singleton(topicPartition));
+ return ProcessContinuation.resume()
+ .withResumeDelay(org.joda.time.Duration.standardSeconds(10L));
}
+ skippedRecords++;
+ continue;
+ }
+ if (skippedRecords > 0L) {
+ LOG.warn(
+ "{} records were skipped due to seek returning an"
+ + " earlier position than requested position of {}",
+ skippedRecords,
+ expectedOffset);
+ skippedRecords = 0L;
}
+ if (!tracker.tryClaim(rawRecord.offset())) {
+ consumer.seek(topicPartition, rawRecord.offset());
+ consumer.pause(Collections.singleton(topicPartition));
- // Non-visible progress within the consumer polling timeout.
- // Claim up to the current position.
- if (expectedOffset < (expectedOffset = consumer.position(topicPartition))) {
- if (!tracker.tryClaim(expectedOffset - 1)) {
- return ProcessContinuation.stop();
+ return ProcessContinuation.stop();
+ }
+ expectedOffset = rawRecord.offset() + 1;
+ try {
+ KafkaRecord<K, V> kafkaRecord =
+ new KafkaRecord<>(
+ rawRecord.topic(),
+ rawRecord.partition(),
+ rawRecord.offset(),
+ ConsumerSpEL.getRecordTimestamp(rawRecord),
+ ConsumerSpEL.getRecordTimestampType(rawRecord),
+ ConsumerSpEL.hasHeaders() ? rawRecord.headers() : null,
+ ConsumerSpEL.deserializeKey(keyDeserializerInstance, rawRecord),
+ ConsumerSpEL.deserializeValue(valueDeserializerInstance, rawRecord));
+ int recordSize =
+ (rawRecord.key() == null ? 0 : rawRecord.key().length)
+ + (rawRecord.value() == null ? 0 : rawRecord.value().length);
+ avgRecordSize.update(recordSize);
+ rawSizes.update(recordSize);
+ Instant outputTimestamp;
+ // The outputTimestamp and watermark will be computed by timestampPolicy, where the
+ // WatermarkEstimator should be a manual one.
+ if (timestampPolicy != null) {
+ TimestampPolicyContext context =
+ updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
+ outputTimestamp = timestampPolicy.getTimestampForRecord(context, kafkaRecord);
+ } else {
+ Preconditions.checkStateNotNull(this.extractOutputTimestampFn);
+ outputTimestamp = extractOutputTimestampFn.apply(kafkaRecord);
}
+ receiver
+ .get(recordTag)
+ .outputWithTimestamp(KV.of(kafkaSourceDescriptor, kafkaRecord), outputTimestamp);
+ } catch (SerializationException e) {
+ // This exception should only occur during the key and value deserialization when
+ // creating the Kafka Record
+ badRecordRouter.route(
+ receiver,
+ rawRecord,
+ null,
+ e,
+ "Failure deserializing Key or Value of Kakfa record reading from Kafka");
if (timestampPolicy != null) {
updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
}
}
+ }
+
+ // Non-visible progress within the consumer polling timeout.
+ // Claim up to the current position.
+ if (expectedOffset < (expectedOffset = consumer.position(topicPartition))) {
+ if (!tracker.tryClaim(expectedOffset - 1)) {
+ consumer.seek(topicPartition, expectedOffset - 1);
+ consumer.pause(Collections.singleton(topicPartition));
- backlogBytes.set(
- (long)
- (BigDecimal.valueOf(
- Preconditions.checkStateNotNull(
- offsetEstimatorCache.get(kafkaSourceDescriptor).estimate()))
- .subtract(BigDecimal.valueOf(expectedOffset), MathContext.DECIMAL128)
- .doubleValue()
- * avgRecordSize.get()));
- kafkaMetrics.updateBacklogBytes(
- kafkaSourceDescriptor.getTopic(),
- kafkaSourceDescriptor.getPartition(),
- (long)
- (BigDecimal.valueOf(
- Preconditions.checkStateNotNull(
- offsetEstimatorCache.get(kafkaSourceDescriptor).estimate()))
- .subtract(BigDecimal.valueOf(expectedOffset), MathContext.DECIMAL128)
- .doubleValue()
- * avgRecordSize.get()));
+ return ProcessContinuation.stop();
+ }
+ if (timestampPolicy != null) {
+ updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
+ }
}
- } finally {
- kafkaMetrics.flushBufferedMetrics();
+
+ final long estimatedBacklogBytes =
+ (long)
+ (BigDecimal.valueOf(latestOffsetEstimator.estimate())
+ .subtract(BigDecimal.valueOf(expectedOffset), MathContext.DECIMAL128)
+ .doubleValue()
+ * avgRecordSize.get());
+ backlogBytes.set(estimatedBacklogBytes);
+ kafkaMetrics.updateBacklogBytes(
+ kafkaSourceDescriptor.getTopic(),
+ kafkaSourceDescriptor.getPartition(),
+ estimatedBacklogBytes);
}
+ } finally {
+ kafkaMetrics.flushBufferedMetrics();
}
}
@@ -621,58 +768,8 @@ abstract class ReadFromKafkaDoFn<K, V>
@Setup
public void setup() throws Exception {
- // Start to track record size.
- avgRecordSizeCache =
- SharedStateHolder.AVG_RECORD_SIZE_CACHE.computeIfAbsent(
- fnId,
- k -> {
- return CacheBuilder.newBuilder()
- .maximumSize(1000L)
- .build(
- new CacheLoader<KafkaSourceDescriptor, MovingAvg>() {
- @Override
- public MovingAvg load(KafkaSourceDescriptor kafkaSourceDescriptor)
- throws Exception {
- return new MovingAvg();
- }
- });
- });
keyDeserializerInstance = keyDeserializerProvider.getDeserializer(consumerConfig, true);
valueDeserializerInstance = valueDeserializerProvider.getDeserializer(consumerConfig, false);
- offsetEstimatorCache =
- SharedStateHolder.OFFSET_ESTIMATOR_CACHE.computeIfAbsent(
- fnId,
- k -> {
- final Map<String, Object> consumerConfig = ImmutableMap.copyOf(this.consumerConfig);
- final @Nullable Map<String, Object> offsetConsumerConfig =
- this.offsetConsumerConfig == null
- ? null
- : ImmutableMap.copyOf(this.offsetConsumerConfig);
- return CacheBuilder.newBuilder()
- .weakValues()
- .expireAfterAccess(1, TimeUnit.MINUTES)
- .build(
- new CacheLoader<KafkaSourceDescriptor, KafkaLatestOffsetEstimator>() {
- @Override
- public KafkaLatestOffsetEstimator load(
- KafkaSourceDescriptor kafkaSourceDescriptor) throws Exception {
- LOG.info(
- "Creating Kafka consumer for offset estimation for {}",
- kafkaSourceDescriptor);
-
- TopicPartition topicPartition = kafkaSourceDescriptor.getTopicPartition();
- Map<String, Object> updatedConsumerConfig =
- overrideBootstrapServersConfig(consumerConfig, kafkaSourceDescriptor);
- Consumer<byte[], byte[]> offsetConsumer =
- consumerFactoryFn.apply(
- KafkaIOUtils.getOffsetConsumerConfig(
- "tracker-" + topicPartition,
- offsetConsumerConfig,
- updatedConsumerConfig));
- return new KafkaLatestOffsetEstimator(offsetConsumer, topicPartition);
- }
- });
- });
if (checkStopReadingFn != null) {
checkStopReadingFn.setup();
}
@@ -680,10 +777,6 @@ abstract class ReadFromKafkaDoFn<K, V>
@Teardown
public void teardown() throws Exception {
- final LoadingCache<KafkaSourceDescriptor, MovingAvg> avgRecordSizeCache =
- Preconditions.checkStateNotNull(this.avgRecordSizeCache);
- final LoadingCache<KafkaSourceDescriptor, KafkaLatestOffsetEstimator> offsetEstimatorCache =
- Preconditions.checkStateNotNull(this.offsetEstimatorCache);
try {
if (valueDeserializerInstance != null) {
Closeables.close(valueDeserializerInstance, true);
@@ -701,22 +794,9 @@ abstract class ReadFromKafkaDoFn<K, V>
}
// Allow the cache to perform clean up tasks when this instance is about to be deleted.
- avgRecordSizeCache.cleanUp();
- offsetEstimatorCache.cleanUp();
- }
-
- private Map<String, Object> overrideBootstrapServersConfig(
- Map<String, Object> currentConfig, KafkaSourceDescriptor description) {
- checkState(
- currentConfig.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)
- || description.getBootStrapServers() != null);
- Map<String, Object> config = new HashMap<>(currentConfig);
- if (description.getBootStrapServers() != null && description.getBootStrapServers().size() > 0) {
- config.put(
- ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
- String.join(",", description.getBootStrapServers()));
- }
- return config;
+ avgRecordSizeCacheSupplier.get().cleanUp();
+ latestOffsetEstimatorCacheSupplier.get().cleanUp();
+ pollConsumerCacheSupplier.get().cleanUp();
}
private static Instant ensureTimestampWithinBounds(Instant timestamp) {
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
index 18aeaac693a..24ed6f9a7e7 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
@@ -326,7 +326,8 @@ public class KafkaIOIT {
.withKeySerializer(IntegerSerializer.class)
.withValueSerializer(StringSerializer.class));
- wPipeline.run().waitUntilFinish(Duration.standardSeconds(10));
+ final PipelineResult wResult = wPipeline.run();
+ cancelIfTimeouted(wResult, wResult.waitUntilFinish(Duration.standardSeconds(10)));
rPipeline
.apply(
@@ -349,7 +350,9 @@ public class KafkaIOIT {
.apply(ParDo.of(new CrashOnExtra(records.values())))
.apply(ParDo.of(new LogFn()));
- rPipeline.run().waitUntilFinish(Duration.standardSeconds(options.getReadTimeout()));
+ final PipelineResult rResult = rPipeline.run();
+ cancelIfTimeouted(
+ rResult, rResult.waitUntilFinish(Duration.standardSeconds(options.getReadTimeout())));
for (String value : records.values()) {
kafkaIOITExpectedLogs.verifyError(value);
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
index eda9dac7a29..2504188d97e 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
@@ -266,12 +266,14 @@ public class ReadFromKafkaDoFnTest {
@Override
public synchronized void assign(Collection<TopicPartition> partitions) {
assertTrue(Iterables.getOnlyElement(partitions).equals(this.topicPartition));
+ super.assign(partitions);
}
@Override
public synchronized void seek(TopicPartition partition, long offset) {
assertTrue(partition.equals(this.topicPartition));
this.startOffset = offset;
+ super.seek(partition, offset);
}
@Override