You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by sh...@apache.org on 2022/06/21 07:58:44 UTC
[kafka] branch trunk updated: KAFKA-10000: Per-connector offsets topics (#11781)
This is an automated email from the ASF dual-hosted git repository.
showuon pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new b7b7615db3 KAFKA-10000: Per-connector offsets topics (#11781)
b7b7615db3 is described below
commit b7b7615db38d8a5d16fa13a227c4a09b4ea355e8
Author: Chris Egerton <ch...@confluent.io>
AuthorDate: Tue Jun 21 03:58:17 2022 -0400
KAFKA-10000: Per-connector offsets topics (#11781)
Implements support for per-connector offsets topics as described in KIP-618.
Reviewers: Luke Chen <sh...@gmail.com>, Tom Bentley <tb...@redhat.com>
---
.../connect/runtime/AbstractWorkerSourceTask.java | 10 +-
.../runtime/ExactlyOnceWorkerSourceTask.java | 26 +-
.../org/apache/kafka/connect/runtime/Worker.java | 304 ++++++++++++-
.../kafka/connect/runtime/WorkerConnector.java | 6 +-
.../kafka/connect/runtime/WorkerSourceTask.java | 4 +-
.../apache/kafka/connect/runtime/WorkerTask.java | 4 +-
.../storage/ConnectorOffsetBackingStore.java | 292 ++++++++++++-
.../connect/storage/KafkaConfigBackingStore.java | 4 +-
.../connect/storage/KafkaOffsetBackingStore.java | 162 ++++++-
.../apache/kafka/connect/util/KafkaBasedLog.java | 124 ++++--
.../org/apache/kafka/connect/util/TopicAdmin.java | 47 +-
.../connect/runtime/ErrorHandlingTaskTest.java | 6 +-
.../runtime/ExactlyOnceWorkerSourceTaskTest.java | 7 +-
.../kafka/connect/runtime/WorkerConnectorTest.java | 18 +
.../connect/runtime/WorkerSourceTaskTest.java | 69 +--
.../apache/kafka/connect/runtime/WorkerTest.java | 472 ++++++++++++++++++++-
.../storage/KafkaOffsetBackingStoreTest.java | 85 ++++
17 files changed, 1459 insertions(+), 181 deletions(-)
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java
index 3a805bdd7d..837891071a 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java
@@ -175,6 +175,7 @@ public abstract class AbstractWorkerSourceTask extends WorkerTask {
protected final WorkerConfig workerConfig;
protected final WorkerSourceTaskContext sourceTaskContext;
+ protected final ConnectorOffsetBackingStore offsetStore;
protected final OffsetStorageWriter offsetWriter;
protected final Producer<byte[], byte[]> producer;
@@ -185,7 +186,6 @@ public abstract class AbstractWorkerSourceTask extends WorkerTask {
private final TransformationChain<SourceRecord> transformationChain;
private final TopicAdmin admin;
private final CloseableOffsetStorageReader offsetReader;
- private final ConnectorOffsetBackingStore offsetStore;
private final SourceTaskMetricsGroup sourceTaskMetricsGroup;
private final CountDownLatch stopRequestedLatch;
private final boolean topicTrackingEnabled;
@@ -256,6 +256,7 @@ public abstract class AbstractWorkerSourceTask extends WorkerTask {
@Override
protected void initializeAndStart() {
prepareToInitializeTask();
+ offsetStore.start();
// If we try to start the task at all by invoking initialize, then count this as
// "started" and expect a subsequent call to the task's stop() method
// to properly clean up any resources allocated by its initialize() or
@@ -291,7 +292,7 @@ public abstract class AbstractWorkerSourceTask extends WorkerTask {
@Override
public void removeMetrics() {
- Utils.closeQuietly(sourceTaskMetricsGroup::close, "source task metrics tracker");
+ Utils.closeQuietly(sourceTaskMetricsGroup, "source task metrics tracker");
super.removeMetrics();
}
@@ -583,7 +584,7 @@ public abstract class AbstractWorkerSourceTask extends WorkerTask {
}
}
- static class SourceTaskMetricsGroup {
+ static class SourceTaskMetricsGroup implements AutoCloseable {
private final ConnectMetrics.MetricGroup metricGroup;
private final Sensor sourceRecordPoll;
private final Sensor sourceRecordWrite;
@@ -617,7 +618,8 @@ public abstract class AbstractWorkerSourceTask extends WorkerTask {
sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountAvg), new Avg());
}
- void close() {
+ @Override
+ public void close() {
metricGroup.close();
}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java
index faf0e8cbd8..de78e592aa 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java
@@ -25,6 +25,7 @@ import org.apache.kafka.common.metrics.stats.Avg;
import org.apache.kafka.common.metrics.stats.Max;
import org.apache.kafka.common.metrics.stats.Min;
import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator;
import org.apache.kafka.connect.source.SourceRecord;
@@ -64,7 +65,6 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
private final TransactionBoundaryManager transactionBoundaryManager;
private final TransactionMetricsGroup transactionMetrics;
- private final ConnectorOffsetBackingStore offsetBackingStore;
private final Runnable preProducerCheck;
private final Runnable postProducerCheck;
@@ -81,7 +81,7 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
Map<String, TopicCreationGroup> topicGroups,
CloseableOffsetStorageReader offsetReader,
OffsetStorageWriter offsetWriter,
- ConnectorOffsetBackingStore offsetBackingStore,
+ ConnectorOffsetBackingStore offsetStore,
WorkerConfig workerConfig,
ClusterConfigState configState,
ConnectMetrics connectMetrics,
@@ -95,12 +95,11 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
Runnable postProducerCheck) {
super(id, task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain,
new WorkerSourceTaskContext(offsetReader, id, configState, buildTransactionContext(sourceConfig)),
- producer, admin, topicGroups, offsetReader, offsetWriter, offsetBackingStore, workerConfig, connectMetrics,
+ producer, admin, topicGroups, offsetReader, offsetWriter, offsetStore, workerConfig, connectMetrics,
loader, time, retryWithToleranceOperator, statusBackingStore, closeExecutor);
this.transactionOpen = false;
this.commitableRecords = new LinkedHashMap<>();
- this.offsetBackingStore = offsetBackingStore;
this.preProducerCheck = preProducerCheck;
this.postProducerCheck = postProducerCheck;
@@ -119,11 +118,6 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
protected void prepareToInitializeTask() {
preProducerCheck.run();
- // Try not to start up the offset store (which has its own producer and consumer) if we've already been shut down at this point
- if (isStopping())
- return;
- offsetBackingStore.start();
-
// Try not to initialize the transactional producer (which may accidentally fence out other, later task generations) if we've already
// been shut down at this point
if (isStopping())
@@ -161,7 +155,7 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
SourceRecord sourceRecord,
ProducerRecord<byte[], byte[]> producerRecord
) {
- if (offsetBackingStore.primaryOffsetsTopic().equals(producerRecord.topic())) {
+ if (offsetStore.primaryOffsetsTopic().equals(producerRecord.topic())) {
// This is to prevent deadlock that occurs when:
// 1. A task provides a record whose topic is the task's offsets topic
// 2. That record is dispatched to the task's producer in a transaction that remains open
@@ -234,6 +228,11 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
transactionBoundaryManager.maybeCommitFinalTransaction();
}
+ @Override
+ public void removeMetrics() {
+ Utils.closeQuietly(transactionMetrics, "source task transaction metrics tracker");
+ }
+
@Override
protected void onPause() {
super.onPause();
@@ -483,7 +482,7 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
}
- static class TransactionMetricsGroup {
+ static class TransactionMetricsGroup implements AutoCloseable {
private final Sensor transactionSize;
private int size;
private final ConnectMetrics.MetricGroup metricGroup;
@@ -500,6 +499,11 @@ class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask {
transactionSize.add(metricGroup.metricName(registry.transactionSizeMax), new Max());
}
+ @Override
+ public void close() {
+ metricGroup.close();
+ }
+
void addRecord() {
size++;
}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
index d06a06a73a..01c0389cae 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
@@ -22,10 +22,12 @@ import org.apache.kafka.clients.admin.FenceProducersOptions;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.MetricNameTemplate;
+import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigValue;
import org.apache.kafka.common.config.provider.ConfigProvider;
import org.apache.kafka.common.utils.Time;
@@ -59,6 +61,7 @@ import org.apache.kafka.connect.storage.CloseableOffsetStorageReader;
import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore;
import org.apache.kafka.connect.storage.Converter;
import org.apache.kafka.connect.storage.HeaderConverter;
+import org.apache.kafka.connect.storage.KafkaOffsetBackingStore;
import org.apache.kafka.connect.storage.OffsetBackingStore;
import org.apache.kafka.connect.storage.OffsetStorageReaderImpl;
import org.apache.kafka.connect.storage.OffsetStorageWriter;
@@ -76,6 +79,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -91,6 +95,8 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+
/**
* <p>
* Worker runs a (dynamic) set of tasks in a set of threads, doing the work of actually moving
@@ -305,8 +311,9 @@ public class Worker {
connConfig = sourceConfig;
// Set up the offset backing store for this connector instance
- // (This logic is implemented in a follow-up pull request: https://github.com/apache/kafka/pull/11781)
- offsetStore = new ConnectorOffsetBackingStore(globalOffsetBackingStore, "TODO");
+ offsetStore = config.exactlyOnceSourceEnabled()
+ ? offsetStoreForExactlyOnceSourceConnector(sourceConfig, connName, connector)
+ : offsetStoreForRegularSourceConnector(sourceConfig, connName, connector);
offsetStore.configure(config);
offsetReader = new OffsetStorageReaderImpl(offsetStore, connName, internalKeyConverter, internalValueConverter);
}
@@ -1327,22 +1334,21 @@ public class Worker {
connectorClientConfigOverridePolicy, kafkaClusterId);
KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(producerProps);
- TopicAdmin topicAdmin;
- Map<String, TopicCreationGroup> topicCreationGroups;
- if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) {
- topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig);
- // Create a topic admin that the task can use for topic creation
+ TopicAdmin topicAdmin = null;
+ final boolean topicCreationEnabled = sourceConnectorTopicCreationEnabled(sourceConfig);
+ if (topicCreationEnabled || regularSourceTaskUsesConnectorSpecificOffsetsStore(sourceConfig)) {
Map<String, Object> adminOverrides = adminConfigs(id.connector(), "connector-adminclient-" + id, config,
sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE);
topicAdmin = new TopicAdmin(adminOverrides);
- } else {
- topicAdmin = null;
- topicCreationGroups = null;
}
+ Map<String, TopicCreationGroup> topicCreationGroups = topicCreationEnabled
+ ? TopicCreationGroup.configuredGroups(sourceConfig)
+ : null;
+
// Set up the offset backing store for this task instance
- // (This logic is implemented in a follow-up pull request: https://github.com/apache/kafka/pull/11781)
- ConnectorOffsetBackingStore offsetStore = new ConnectorOffsetBackingStore(globalOffsetBackingStore, "TODO");
+ ConnectorOffsetBackingStore offsetStore = offsetStoreForRegularSourceTask(
+ id, sourceConfig, connectorClass, producer, producerProps, topicAdmin);
offsetStore.configure(config);
CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, id.connector(), internalKeyConverter, internalValueConverter);
@@ -1402,16 +1408,13 @@ public class Worker {
sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE);
TopicAdmin topicAdmin = new TopicAdmin(adminOverrides);
- Map<String, TopicCreationGroup> topicCreationGroups;
- if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) {
- topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig);
- } else {
- topicCreationGroups = null;
- }
+ Map<String, TopicCreationGroup> topicCreationGroups = sourceConnectorTopicCreationEnabled(sourceConfig)
+ ? TopicCreationGroup.configuredGroups(sourceConfig)
+ : null;
// Set up the offset backing store for this task instance
- // (This logic is implemented in a follow-up pull request: https://github.com/apache/kafka/pull/11781)
- ConnectorOffsetBackingStore offsetStore = new ConnectorOffsetBackingStore(globalOffsetBackingStore, "TODO");
+ ConnectorOffsetBackingStore offsetStore = offsetStoreForExactlyOnceSourceTask(
+ id, sourceConfig, connectorClass, producer, producerProps, topicAdmin);
offsetStore.configure(config);
CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, id.connector(), internalKeyConverter, internalValueConverter);
@@ -1425,6 +1428,267 @@ public class Worker {
}
}
+ // Visible for testing
+ ConnectorOffsetBackingStore offsetStoreForRegularSourceConnector(
+ SourceConnectorConfig sourceConfig,
+ String connName,
+ Connector connector
+ ) {
+ String connectorSpecificOffsetsTopic = sourceConfig.offsetsTopic();
+
+ Map<String, Object> producerProps = baseProducerConfigs(connName, "connector-producer-" + connName, config, sourceConfig, connector.getClass(),
+ connectorClientConfigOverridePolicy, kafkaClusterId);
+
+ // We use a connector-specific store (i.e., a dedicated KafkaOffsetBackingStore for this connector)
+ // if the worker supports per-connector offsets topics (which may be the case in distributed but not standalone mode, for example)
+ // and if the connector is explicitly configured with an offsets topic
+ final boolean usesConnectorSpecificStore = connectorSpecificOffsetsTopic != null
+ && config.connectorOffsetsTopicsPermitted();
+
+ if (usesConnectorSpecificStore) {
+ Map<String, Object> consumerProps = regularSourceOffsetsConsumerConfigs(
+ connName, "connector-consumer-" + connName, config, sourceConfig, connector.getClass(),
+ connectorClientConfigOverridePolicy, kafkaClusterId);
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
+
+ Map<String, Object> adminOverrides = adminConfigs(connName, "connector-adminclient-" + connName, config,
+ sourceConfig, connector.getClass(), connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE);
+
+ TopicAdmin admin = new TopicAdmin(adminOverrides);
+ KafkaOffsetBackingStore connectorStore =
+ KafkaOffsetBackingStore.forConnector(connectorSpecificOffsetsTopic, consumer, admin);
+
+ // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct
+ // an offset store that has a primary and a secondary store which both read from that same topic.
+ // So, if the user has explicitly configured the connector with a connector-specific offsets topic
+ // but we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global
+ // offset store and build a store backed exclusively by a connector-specific offsets store.
+ // It may seem reasonable to instead build a store backed exclusively by the worker-global offset store, but that
+ // would prevent users from being able to customize the config properties used for the Kafka clients that
+ // access the offsets topic, and we would not be able to establish reasonable defaults like setting
+ // isolation.level=read_committed for the offsets topic consumer for this connector
+ if (sameOffsetTopicAsWorker(connectorSpecificOffsetsTopic, producerProps)) {
+ return ConnectorOffsetBackingStore.withOnlyConnectorStore(
+ () -> LoggingContext.forConnector(connName),
+ connectorStore,
+ connectorSpecificOffsetsTopic,
+ admin
+ );
+ } else {
+ return ConnectorOffsetBackingStore.withConnectorAndWorkerStores(
+ () -> LoggingContext.forConnector(connName),
+ globalOffsetBackingStore,
+ connectorStore,
+ connectorSpecificOffsetsTopic,
+ admin
+ );
+ }
+ } else {
+ return ConnectorOffsetBackingStore.withOnlyWorkerStore(
+ () -> LoggingContext.forConnector(connName),
+ globalOffsetBackingStore,
+ config.offsetsTopic()
+ );
+ }
+ }
+
+ // Visible for testing
+ ConnectorOffsetBackingStore offsetStoreForExactlyOnceSourceConnector(
+ SourceConnectorConfig sourceConfig,
+ String connName,
+ Connector connector
+ ) {
+ String connectorSpecificOffsetsTopic = Optional.ofNullable(sourceConfig.offsetsTopic()).orElse(config.offsetsTopic());
+
+ Map<String, Object> producerProps = baseProducerConfigs(connName, "connector-producer-" + connName, config, sourceConfig, connector.getClass(),
+ connectorClientConfigOverridePolicy, kafkaClusterId);
+
+ Map<String, Object> consumerProps = exactlyOnceSourceOffsetsConsumerConfigs(
+ connName, "connector-consumer-" + connName, config, sourceConfig, connector.getClass(),
+ connectorClientConfigOverridePolicy, kafkaClusterId);
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
+
+ Map<String, Object> adminOverrides = adminConfigs(connName, "connector-adminclient-" + connName, config,
+ sourceConfig, connector.getClass(), connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE);
+
+ TopicAdmin admin = new TopicAdmin(adminOverrides);
+ KafkaOffsetBackingStore connectorStore =
+ KafkaOffsetBackingStore.forConnector(connectorSpecificOffsetsTopic, consumer, admin);
+
+ // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct
+ // an offset store that has a primary and a secondary store which both read from that same topic.
+ // So, even if the user has explicitly configured the connector with a connector-specific offsets topic,
+ // if we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global
+ // offset store and build a store backed exclusively by a connector-specific offsets store.
+ // It may seem reasonable to instead build a store backed exclusively by the worker-global offset store, but that
+ // would prevent users from being able to customize the config properties used for the Kafka clients that
+ // access the offsets topic, and may lead to confusion for them when tasks are created for the connector
+ // since they will all have their own dedicated offsets stores anyways
+ if (sameOffsetTopicAsWorker(connectorSpecificOffsetsTopic, producerProps)) {
+ return ConnectorOffsetBackingStore.withOnlyConnectorStore(
+ () -> LoggingContext.forConnector(connName),
+ connectorStore,
+ connectorSpecificOffsetsTopic,
+ admin
+ );
+ } else {
+ return ConnectorOffsetBackingStore.withConnectorAndWorkerStores(
+ () -> LoggingContext.forConnector(connName),
+ globalOffsetBackingStore,
+ connectorStore,
+ connectorSpecificOffsetsTopic,
+ admin
+ );
+ }
+ }
+
+ // Visible for testing
+ ConnectorOffsetBackingStore offsetStoreForRegularSourceTask(
+ ConnectorTaskId id,
+ SourceConnectorConfig sourceConfig,
+ Class<? extends Connector> connectorClass,
+ Producer<byte[], byte[]> producer,
+ Map<String, Object> producerProps,
+ TopicAdmin topicAdmin
+ ) {
+ String connectorSpecificOffsetsTopic = sourceConfig.offsetsTopic();
+
+ if (regularSourceTaskUsesConnectorSpecificOffsetsStore(sourceConfig)) {
+ Objects.requireNonNull(topicAdmin, "Source tasks require a non-null topic admin when configured to use their own offsets topic");
+
+ Map<String, Object> consumerProps = regularSourceOffsetsConsumerConfigs(
+ id.connector(), "connector-consumer-" + id, config, sourceConfig, connectorClass,
+ connectorClientConfigOverridePolicy, kafkaClusterId);
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
+
+ KafkaOffsetBackingStore connectorStore =
+ KafkaOffsetBackingStore.forTask(sourceConfig.offsetsTopic(), producer, consumer, topicAdmin);
+
+ // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct
+ // an offset store that has a primary and a secondary store which both read from that same topic.
+ // So, if the user has (implicitly or explicitly) configured the connector with a connector-specific offsets topic
+ // but we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global
+ // offset store and build a store backed exclusively by a connector-specific offsets store.
+ // It may seem reasonable to instead build a store backed exclusively by the worker-global offset store, but that
+ // would prevent users from being able to customize the config properties used for the Kafka clients that
+ // access the offsets topic, and we would not be able to establish reasonable defaults like setting
+ // isolation.level=read_committed for the offsets topic consumer for this task
+ if (sameOffsetTopicAsWorker(sourceConfig.offsetsTopic(), producerProps)) {
+ return ConnectorOffsetBackingStore.withOnlyConnectorStore(
+ () -> LoggingContext.forTask(id),
+ connectorStore,
+ connectorSpecificOffsetsTopic,
+ topicAdmin
+ );
+ } else {
+ return ConnectorOffsetBackingStore.withConnectorAndWorkerStores(
+ () -> LoggingContext.forTask(id),
+ globalOffsetBackingStore,
+ connectorStore,
+ connectorSpecificOffsetsTopic,
+ topicAdmin
+ );
+ }
+ } else {
+ return ConnectorOffsetBackingStore.withOnlyWorkerStore(
+ () -> LoggingContext.forTask(id),
+ globalOffsetBackingStore,
+ config.offsetsTopic()
+ );
+ }
+ }
+
+ // Visible for testing
+ ConnectorOffsetBackingStore offsetStoreForExactlyOnceSourceTask(
+ ConnectorTaskId id,
+ SourceConnectorConfig sourceConfig,
+ Class<? extends Connector> connectorClass,
+ Producer<byte[], byte[]> producer,
+ Map<String, Object> producerProps,
+ TopicAdmin topicAdmin
+ ) {
+ Objects.requireNonNull(topicAdmin, "Source tasks require a non-null topic admin when exactly-once support is enabled");
+
+ Map<String, Object> consumerProps = exactlyOnceSourceOffsetsConsumerConfigs(
+ id.connector(), "connector-consumer-" + id, config, sourceConfig, connectorClass,
+ connectorClientConfigOverridePolicy, kafkaClusterId);
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
+
+ String connectorOffsetsTopic = Optional.ofNullable(sourceConfig.offsetsTopic()).orElse(config.offsetsTopic());
+
+ KafkaOffsetBackingStore connectorStore =
+ KafkaOffsetBackingStore.forTask(connectorOffsetsTopic, producer, consumer, topicAdmin);
+
+ // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct
+ // an offset store that has a primary and a secondary store which both read from that same topic.
+ // So, if the user has (implicitly or explicitly) configured the connector with a connector-specific offsets topic
+ // but we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global
+ // offset store and build a store backed exclusively by a connector-specific offsets store.
+ // We cannot under any circumstances build an offset store backed exclusively by the worker-global offset store
+ // as that would prevent us from being able to write source records and source offset information for the task
+ // with the same producer, and therefore, in the same transaction.
+ if (sameOffsetTopicAsWorker(connectorOffsetsTopic, producerProps)) {
+ return ConnectorOffsetBackingStore.withOnlyConnectorStore(
+ () -> LoggingContext.forTask(id),
+ connectorStore,
+ connectorOffsetsTopic,
+ topicAdmin
+ );
+ } else {
+ return ConnectorOffsetBackingStore.withConnectorAndWorkerStores(
+ () -> LoggingContext.forTask(id),
+ globalOffsetBackingStore,
+ connectorStore,
+ connectorOffsetsTopic,
+ topicAdmin
+ );
+ }
+ }
+
+ /**
+ * Gives a best-effort guess for whether the given offsets topic is the same topic as the worker-global offsets topic.
+ * Even if the name of the topic is the same as the name of the worker's offsets topic, the two may still be different topics
+ * if the connector is configured to produce to a different Kafka cluster than the one that hosts the worker's offsets topic.
+ * @param offsetsTopic the name of the offsets topic for the connector
+ * @param producerProps the producer configuration for the connector
+ * @return whether it appears that the connector's offsets topic is the same topic as the worker-global offsets topic.
+ * If {@code true}, it is guaranteed that the two are the same;
+ * if {@code false}, it is likely but not guaranteed that the two are not the same
+ */
+ private boolean sameOffsetTopicAsWorker(String offsetsTopic, Map<String, Object> producerProps) {
+ // We can check the offset topic name and the Kafka cluster's bootstrap servers,
+ // although this isn't exact and can lead to some false negatives if the user
+ // provides an overridden bootstrap servers value for their producer that is different than
+ // the worker's but still resolves to the same Kafka cluster used by the worker.
+ // At the moment this is probably adequate, especially since we don't want to put
+ // a network ping to a remote Kafka cluster inside the herder's tick thread (which is where this
+ // logic takes place right now) in case that takes a while.
+ Set<String> workerBootstrapServers = new HashSet<>(config.getList(BOOTSTRAP_SERVERS_CONFIG));
+ Set<String> producerBootstrapServers = new HashSet<>();
+ try {
+ String rawBootstrapServers = producerProps.getOrDefault(BOOTSTRAP_SERVERS_CONFIG, "").toString();
+ @SuppressWarnings("unchecked")
+ List<String> parsedBootstrapServers = (List<String>) ConfigDef.parseType(BOOTSTRAP_SERVERS_CONFIG, rawBootstrapServers, ConfigDef.Type.LIST);
+ producerBootstrapServers.addAll(parsedBootstrapServers);
+ } catch (Exception e) {
+ // Should never happen by this point, but if it does, make sure to present a readable error message to the user
+ throw new ConnectException("Failed to parse bootstrap servers property in producer config", e);
+ }
+ return offsetsTopic.equals(config.offsetsTopic())
+ && workerBootstrapServers.equals(producerBootstrapServers);
+ }
+
+ private boolean regularSourceTaskUsesConnectorSpecificOffsetsStore(SourceConnectorConfig sourceConfig) {
+ // We use a connector-specific store (i.e., a dedicated KafkaOffsetBackingStore for this task)
+ // if the worker supports per-connector offsets topics (which may be the case in distributed mode but not standalone, for example)
+ // and the user has explicitly specified an offsets topic for the connector
+ return sourceConfig.offsetsTopic() != null && config.connectorOffsetsTopicsPermitted();
+ }
+
+ private boolean sourceConnectorTopicCreationEnabled(SourceConnectorConfig sourceConfig) {
+ return config.topicCreationEnable() && sourceConfig.usesTopicCreation();
+ }
+
static class ConnectorStatusMetricsGroup {
private final ConnectMetrics connectMetrics;
private final ConnectMetricsRegistry registry;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
index dfa13103fa..fa3acfb88f 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java
@@ -171,7 +171,9 @@ public class WorkerConnector implements Runnable {
SinkConnectorConfig.validate(config);
connector.initialize(new WorkerSinkConnectorContext());
} else {
+ Objects.requireNonNull(offsetStore, "Offset store cannot be null for source connectors");
Objects.requireNonNull(offsetStorageReader, "Offset reader cannot be null for source connectors");
+ offsetStore.start();
connector.initialize(new WorkerSourceConnectorContext(offsetStorageReader));
}
} catch (Throwable t) {
@@ -281,7 +283,9 @@ public class WorkerConnector implements Runnable {
Utils.closeQuietly(ctx, "connector context for " + connName);
Utils.closeQuietly(metrics, "connector metrics for " + connName);
Utils.closeQuietly(offsetStorageReader, "offset reader for " + connName);
- Utils.closeQuietly(offsetStore::stop, "offset backing store for " + connName);
+ if (offsetStore != null) {
+ Utils.closeQuietly(offsetStore::stop, "offset backing store for " + connName);
+ }
}
}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
index 0b8c7fa52e..1a63cc1532 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java
@@ -73,7 +73,7 @@ class WorkerSourceTask extends AbstractWorkerSourceTask {
Map<String, TopicCreationGroup> topicGroups,
CloseableOffsetStorageReader offsetReader,
OffsetStorageWriter offsetWriter,
- ConnectorOffsetBackingStore offsetBackingStore,
+ ConnectorOffsetBackingStore offsetStore,
WorkerConfig workerConfig,
ClusterConfigState configState,
ConnectMetrics connectMetrics,
@@ -85,7 +85,7 @@ class WorkerSourceTask extends AbstractWorkerSourceTask {
super(id, task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain,
new WorkerSourceTaskContext(offsetReader, id, configState, null), producer,
- admin, topicGroups, offsetReader, offsetWriter, offsetBackingStore, workerConfig, connectMetrics, loader,
+ admin, topicGroups, offsetReader, offsetWriter, offsetStore, workerConfig, connectMetrics, loader,
time, retryWithToleranceOperator, statusBackingStore, closeExecutor);
this.committableOffsets = CommittableOffsets.EMPTY;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java
index f7c819fb4a..ea086199aa 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java
@@ -53,10 +53,10 @@ abstract class WorkerTask implements Runnable {
private static final Logger log = LoggerFactory.getLogger(WorkerTask.class);
private static final String THREAD_NAME_PREFIX = "task-thread-";
- protected final ConnectorTaskId id;
private final TaskStatus.Listener statusListener;
- protected final ClassLoader loader;
private final StatusBackingStore statusBackingStore;
+ protected final ConnectorTaskId id;
+ protected final ClassLoader loader;
protected final Time time;
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
private final TaskMetricsGroup taskMetricsGroup;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java
index 8f0f068bc0..b33315b9f3 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java
@@ -16,56 +16,326 @@
*/
package org.apache.kafka.connect.storage;
+import org.apache.kafka.common.utils.Time;
import org.apache.kafka.connect.runtime.WorkerConfig;
import org.apache.kafka.connect.util.Callback;
+import org.apache.kafka.connect.util.LoggingContext;
+import org.apache.kafka.connect.util.TopicAdmin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
+import java.time.Duration;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+/**
+ * An {@link OffsetBackingStore} with support for reading from and writing to a worker-global
+ * offset backing store and/or a connector-specific offset backing store.
+ */
public class ConnectorOffsetBackingStore implements OffsetBackingStore {
- private final OffsetBackingStore workerStore;
+ private static final Logger log = LoggerFactory.getLogger(ConnectorOffsetBackingStore.class);
+
+ /**
+ * Builds an offset store that uses a connector-specific offset topic as the primary store and
+ * the worker-global offset store as the secondary store.
+ *
+ * @param loggingContext a {@link Supplier} for the {@link LoggingContext} that should be used
+ * for messages logged by this offset store; may not be null, and may never return null
+ * @param workerStore the worker-global offset store; may not be null
+ * @param connectorStore the connector-specific offset store; may not be null
+ * @param connectorOffsetsTopic the name of the connector-specific offset topic; may not be null
+ * @param connectorStoreAdmin the topic admin to use for the connector-specific offset topic; may not be null
+ * @return an offset store backed primarily by the connector-specific offset topic and secondarily
+ * by the worker-global offset store; never null
+ */
+ public static ConnectorOffsetBackingStore withConnectorAndWorkerStores(
+ Supplier<LoggingContext> loggingContext,
+ OffsetBackingStore workerStore,
+ KafkaOffsetBackingStore connectorStore,
+ String connectorOffsetsTopic,
+ TopicAdmin connectorStoreAdmin
+ ) {
+ Objects.requireNonNull(loggingContext);
+ Objects.requireNonNull(workerStore);
+ Objects.requireNonNull(connectorStore);
+ Objects.requireNonNull(connectorOffsetsTopic);
+ Objects.requireNonNull(connectorStoreAdmin);
+ return new ConnectorOffsetBackingStore(
+ Time.SYSTEM,
+ loggingContext,
+ connectorOffsetsTopic,
+ workerStore,
+ connectorStore,
+ connectorStoreAdmin
+ );
+ }
+
+ /**
+ * Builds an offset store that uses the worker-global offset store as the primary store, and no secondary store.
+ *
+ * @param loggingContext a {@link Supplier} for the {@link LoggingContext} that should be used
+ * for messages logged by this offset store; may not be null, and may never return null
+ * @param workerStore the worker-global offset store; may not be null
+ * @param workerOffsetsTopic the name of the worker-global offset topic; may be null if the worker
+ * does not use an offset topic for its offset store
+ * @return an offset store for the connector backed solely by the worker-global offset store; never null
+ */
+ public static ConnectorOffsetBackingStore withOnlyWorkerStore(
+ Supplier<LoggingContext> loggingContext,
+ OffsetBackingStore workerStore,
+ String workerOffsetsTopic
+ ) {
+ Objects.requireNonNull(loggingContext);
+ Objects.requireNonNull(workerStore);
+ return new ConnectorOffsetBackingStore(Time.SYSTEM, loggingContext, workerOffsetsTopic, workerStore, null, null);
+ }
+
+ /**
+ * Builds an offset store that uses a connector-specific offset topic as the primary store, and no secondary store.
+ *
+ * @param loggingContext a {@link Supplier} for the {@link LoggingContext} that should be used
+ * for messages logged by this offset store; may not be null, and may never return null
+ * @param connectorStore the connector-specific offset store; may not be null
+ * @param connectorOffsetsTopic the name of the connector-specific offset topic; may not be null
+ * @param connectorStoreAdmin the topic admin to use for the connector-specific offset topic; may not be null
+ * @return an offset store for the connector backed solely by the connector-specific offset topic; never null
+ */
+ public static ConnectorOffsetBackingStore withOnlyConnectorStore(
+ Supplier<LoggingContext> loggingContext,
+ KafkaOffsetBackingStore connectorStore,
+ String connectorOffsetsTopic,
+ TopicAdmin connectorStoreAdmin
+ ) {
+ Objects.requireNonNull(loggingContext);
+ Objects.requireNonNull(connectorOffsetsTopic);
+ Objects.requireNonNull(connectorStoreAdmin);
+ return new ConnectorOffsetBackingStore(
+ Time.SYSTEM,
+ loggingContext,
+ connectorOffsetsTopic,
+ null,
+ connectorStore,
+ connectorStoreAdmin
+ );
+ }
+
+ private final Time time;
+ private final Supplier<LoggingContext> loggingContext;
private final String primaryOffsetsTopic;
+ private final Optional<OffsetBackingStore> workerStore;
+ private final Optional<KafkaOffsetBackingStore> connectorStore;
+ private final Optional<TopicAdmin> connectorStoreAdmin;
- public ConnectorOffsetBackingStore(
+ ConnectorOffsetBackingStore(
+ Time time,
+ Supplier<LoggingContext> loggingContext,
+ String primaryOffsetsTopic,
OffsetBackingStore workerStore,
- String primaryOffsetsTopic
+ KafkaOffsetBackingStore connectorStore,
+ TopicAdmin connectorStoreAdmin
) {
- this.workerStore = workerStore;
+ if (workerStore == null && connectorStore == null) {
+ throw new IllegalArgumentException("At least one non-null offset store must be provided");
+ }
+ this.time = time;
+ this.loggingContext = loggingContext;
this.primaryOffsetsTopic = primaryOffsetsTopic;
+ this.workerStore = Optional.ofNullable(workerStore);
+ this.connectorStore = Optional.ofNullable(connectorStore);
+ this.connectorStoreAdmin = Optional.ofNullable(connectorStoreAdmin);
}
public String primaryOffsetsTopic() {
return primaryOffsetsTopic;
}
+ /**
+ * If configured to use a connector-specific offset store, {@link OffsetBackingStore#start() start} that store.
+ *
+ * <p>The worker-global offset store is not modified; it is the caller's responsibility to ensure that it is started
+ * before calls to {@link #get(Collection)} and {@link #set(Map, Callback)} take place.
+ */
@Override
public void start() {
- // TODO
+ // Worker offset store should already be started
+ connectorStore.ifPresent(OffsetBackingStore::start);
}
+ /**
+ * If configured to use a connector-specific offset store, {@link OffsetBackingStore#stop() stop} that store,
+ * and {@link TopicAdmin#close(Duration) close} the topic admin used by that store.
+ *
+ * <p>The worker-global offset store is not modified as it may be used for other connectors that either already exist,
+ * or will be created, on this worker.
+ */
@Override
public void stop() {
- // TODO
+ // Worker offset store should not be stopped as it may be used for multiple connectors
+ connectorStore.ifPresent(OffsetBackingStore::stop);
+ connectorStoreAdmin.ifPresent(TopicAdmin::close);
}
+ /**
+ * Get the offset values for the specified keys.
+ *
+ * <p>If configured to use a connector-specific offset store, priority is given to the values contained in that store,
+ * and the values in the worker-global offset store (if one is provided) are used as a fallback for keys that are not
+ * present in the connector-specific store.
+ *
+ * <p>If not configured to use a connector-specific offset store, only the values contained in the worker-global
+ * offset store are returned.
+
+ * @param keys list of keys to look up
+ * @return future for the resulting map from key to value
+ */
@Override
public Future<Map<ByteBuffer, ByteBuffer>> get(Collection<ByteBuffer> keys) {
- // TODO
- return workerStore.get(keys);
+ Future<Map<ByteBuffer, ByteBuffer>> workerGetFuture = getFromStore(workerStore, keys);
+ Future<Map<ByteBuffer, ByteBuffer>> connectorGetFuture = getFromStore(connectorStore, keys);
+
+ return new Future<Map<ByteBuffer, ByteBuffer>>() {
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ // Note the use of | instead of || here; this causes cancel to be invoked on both futures,
+ // even if the first call to cancel returns true
+ return workerGetFuture.cancel(mayInterruptIfRunning)
+ | connectorGetFuture.cancel(mayInterruptIfRunning);
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return workerGetFuture.isCancelled()
+ || connectorGetFuture.isCancelled();
+ }
+
+ @Override
+ public boolean isDone() {
+ return workerGetFuture.isDone()
+ && connectorGetFuture.isDone();
+ }
+
+ @Override
+ public Map<ByteBuffer, ByteBuffer> get() throws InterruptedException, ExecutionException {
+ Map<ByteBuffer, ByteBuffer> result = new HashMap<>(workerGetFuture.get());
+ result.putAll(connectorGetFuture.get());
+ return result;
+ }
+
+ @Override
+ public Map<ByteBuffer, ByteBuffer> get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ long timeoutMs = unit.toMillis(timeout);
+ long endTime = time.milliseconds() + timeoutMs;
+ Map<ByteBuffer, ByteBuffer> result = new HashMap<>(workerGetFuture.get(timeoutMs, unit));
+ timeoutMs = Math.max(1, endTime - time.milliseconds());
+ result.putAll(connectorGetFuture.get(timeoutMs, TimeUnit.MILLISECONDS));
+ return result;
+ }
+ };
}
+ /**
+ * Store the specified offset key/value pairs.
+ *
+ * <p>If configured to use a connector-specific offset store, the returned {@link Future} corresponds to a
+ * write to that store, and the passed-in {@link Callback} is invoked once that write completes. If a worker-global
+ * store is provided, a secondary write is made to that store if the write to the connector-specific store
+ * succeeds. Errors with this secondary write are not reflected in the returned {@link Future} or the passed-in
+ * {@link Callback}; they are only logged as a warning to users.
+ *
+ * <p>If not configured to use a connector-specific offset store, the returned {@link Future} corresponds to a
+ * write to the worker-global offset store, and the passed-in {@link Callback} is invoked once that write completes.
+
+ * @param values map from key to value
+ * @param callback callback to invoke on completion of the primary write
+ * @return void future for the primary write
+ */
@Override
public Future<Void> set(Map<ByteBuffer, ByteBuffer> values, Callback<Void> callback) {
- // TODO
- return workerStore.set(values, callback);
+ final OffsetBackingStore primaryStore;
+ final OffsetBackingStore secondaryStore;
+ if (connectorStore.isPresent()) {
+ primaryStore = connectorStore.get();
+ secondaryStore = workerStore.orElse(null);
+ } else if (workerStore.isPresent()) {
+ primaryStore = workerStore.get();
+ secondaryStore = null;
+ } else {
+ // Should never happen since we check for this case in the constructor, but just in case, this should
+ // be more informative than the NPE that would otherwise be thrown
+ throw new IllegalStateException("At least one non-null offset store must be provided");
+ }
+
+ return primaryStore.set(values, (primaryWriteError, ignored) -> {
+ if (secondaryStore != null) {
+ if (primaryWriteError != null) {
+ log.trace("Skipping offsets write to secondary store because primary write has failed", primaryWriteError);
+ } else {
+ try {
+ // Invoke OffsetBackingStore::set but ignore the resulting future; we don't block on writes to this
+ // backing store.
+ secondaryStore.set(values, (secondaryWriteError, ignored2) -> {
+ try (LoggingContext context = loggingContext()) {
+ if (secondaryWriteError != null) {
+ log.warn("Failed to write offsets to secondary backing store", secondaryWriteError);
+ } else {
+ log.debug("Successfully flushed offsets to secondary backing store");
+ }
+ }
+ });
+ } catch (Exception e) {
+ log.warn("Failed to write offsets to secondary backing store", e);
+ }
+ }
+ }
+ try (LoggingContext context = loggingContext()) {
+ callback.onCompletion(primaryWriteError, ignored);
+ }
+ });
}
+ /**
+ * If configured to use a connector-specific offset store,
+ * {@link OffsetBackingStore#configure(WorkerConfig) configure} that store.
+ *
+ * <p>The worker-global offset store is not modified; it is the caller's responsibility to ensure that it is configured
+ * before calls to {@link #start()}, {@link #get(Collection)} and {@link #set(Map, Callback)} take place.
+ */
@Override
public void configure(WorkerConfig config) {
- // TODO
+ // Worker offset store should already be configured
+ connectorStore.ifPresent(store -> store.configure(config));
+ }
+
+ // For testing
+ public boolean hasConnectorSpecificStore() {
+ return connectorStore.isPresent();
+ }
+
+ // For testing
+ public boolean hasWorkerGlobalStore() {
+ return workerStore.isPresent();
+ }
+
+ private LoggingContext loggingContext() {
+ LoggingContext result = loggingContext.get();
+ Objects.requireNonNull(result);
+ return result;
+ }
+
+ private static Future<Map<ByteBuffer, ByteBuffer>> getFromStore(Optional<? extends OffsetBackingStore> store, Collection<ByteBuffer> keys) {
+ return store.map(s -> s.get(keys)).orElseGet(() -> CompletableFuture.completedFuture(Collections.emptyMap()));
}
}
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
index f8467e85b5..76c626964e 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java
@@ -355,9 +355,7 @@ public class KafkaConfigBackingStore implements ConfigBackingStore {
public void stop() {
log.info("Closing KafkaConfigBackingStore");
- if (fencableProducer != null) {
- Utils.closeQuietly(() -> fencableProducer.close(Duration.ZERO), "fencable producer for config topic");
- }
+ relinquishWritePrivileges();
Utils.closeQuietly(ownTopicAdmin, "admin for config topic");
Utils.closeQuietly(configLog::stop, "KafkaBasedLog for config topic");
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java
index f3cbb686ff..6693572002 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java
@@ -17,15 +17,20 @@
package org.apache.kafka.connect.storage;
import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.runtime.WorkerConfig;
import org.apache.kafka.connect.runtime.distributed.DistributedConfig;
import org.apache.kafka.connect.util.Callback;
@@ -38,9 +43,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
+import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -63,28 +70,109 @@ import java.util.function.Supplier;
public class KafkaOffsetBackingStore implements OffsetBackingStore {
private static final Logger log = LoggerFactory.getLogger(KafkaOffsetBackingStore.class);
- private KafkaBasedLog<byte[], byte[]> offsetLog;
- private HashMap<ByteBuffer, ByteBuffer> data;
+ /**
+ * Build a connector-specific offset store with read and write support. The producer will be {@link Producer#close(Duration) closed}
+ * and the consumer will be {@link Consumer#close(Duration) closed} when this store is {@link #stop() stopped}, but the topic admin
+ * must be {@link TopicAdmin#close(Duration) closed} by the caller.
+ * @param topic the name of the offsets topic to use
+ * @param producer the producer to use for writing to the offsets topic
+ * @param consumer the consumer to use for reading from the offsets topic
+ * @param topicAdmin the topic admin to use for creating and querying metadata for the offsets topic
+ * @return an offset store backed by the given topic and Kafka clients
+ */
+ public static KafkaOffsetBackingStore forTask(
+ String topic,
+ Producer<byte[], byte[]> producer,
+ Consumer<byte[], byte[]> consumer,
+ TopicAdmin topicAdmin
+ ) {
+ return new KafkaOffsetBackingStore(() -> topicAdmin) {
+ @Override
+ public void configure(final WorkerConfig config) {
+ this.exactlyOnce = config.exactlyOnceSourceEnabled();
+ this.offsetLog = KafkaBasedLog.withExistingClients(
+ topic,
+ consumer,
+ producer,
+ topicAdmin,
+ consumedCallback,
+ Time.SYSTEM,
+ initialize(topic, newTopicDescription(topic, config))
+ );
+ }
+ };
+ }
+
+ /**
+ * Build a connector-specific offset store with read-only support. The consumer will be {@link Consumer#close(Duration) closed}
+ * when this store is {@link #stop() stopped}, but the topic admin must be {@link TopicAdmin#close(Duration) closed} by the caller.
+ * @param topic the name of the offsets topic to use
+ * @param consumer the consumer to use for reading from the offsets topic
+ * @param topicAdmin the topic admin to use for creating and querying metadata for the offsets topic
+ * @return a read-only offset store backed by the given topic and Kafka clients
+ */
+ public static KafkaOffsetBackingStore forConnector(
+ String topic,
+ Consumer<byte[], byte[]> consumer,
+ TopicAdmin topicAdmin
+ ) {
+ return new KafkaOffsetBackingStore(() -> topicAdmin) {
+ @Override
+ public void configure(final WorkerConfig config) {
+ this.exactlyOnce = config.exactlyOnceSourceEnabled();
+ this.offsetLog = KafkaBasedLog.withExistingClients(
+ topic,
+ consumer,
+ null,
+ topicAdmin,
+ consumedCallback,
+ Time.SYSTEM,
+ initialize(topic, newTopicDescription(topic, config))
+ );
+ }
+ };
+ }
+
+ protected KafkaBasedLog<byte[], byte[]> offsetLog;
+ private final HashMap<ByteBuffer, ByteBuffer> data = new HashMap<>();
private final Supplier<TopicAdmin> topicAdminSupplier;
private SharedTopicAdmin ownTopicAdmin;
+ protected boolean exactlyOnce;
+ /**
+ * Create an {@link OffsetBackingStore} backed by a Kafka topic. This constructor will cause the
+ * store to instantiate and close its own {@link TopicAdmin} during {@link #configure(WorkerConfig)}
+ * and {@link #stop()}, respectively.
+ *
+ * @deprecated use {@link #KafkaOffsetBackingStore(Supplier)} instead
+ */
@Deprecated
public KafkaOffsetBackingStore() {
this.topicAdminSupplier = null;
}
+ /**
+ * Create an {@link OffsetBackingStore} backed by a Kafka topic. This constructor will use the given
+ * {@link Supplier} to acquire a {@link TopicAdmin} that will be used for interactions with the backing
+ * Kafka topic. The caller is expected to manage the lifecycle of that object, including
+ * {@link TopicAdmin#close(Duration) closing} it when it is no longer needed.
+ * @param topicAdmin a {@link Supplier} for the {@link TopicAdmin} to use for this backing store;
+ * may not be null, and may not return null
+ */
public KafkaOffsetBackingStore(Supplier<TopicAdmin> topicAdmin) {
this.topicAdminSupplier = Objects.requireNonNull(topicAdmin);
}
+
@Override
public void configure(final WorkerConfig config) {
String topic = config.getString(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG);
if (topic == null || topic.trim().length() == 0)
throw new ConfigException("Offset storage topic must be specified");
+ this.exactlyOnce = config.exactlyOnceSourceEnabled();
+
String clusterId = ConnectUtils.lookupKafkaClusterId(config);
- data = new HashMap<>();
Map<String, Object> originals = config.originals();
Map<String, Object> producerProps = new HashMap<>(originals);
@@ -103,6 +191,13 @@ public class KafkaOffsetBackingStore implements OffsetBackingStore {
consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId);
+ if (config.exactlyOnceSourceEnabled()) {
+ ConnectUtils.ensureProperty(
+ consumerProps, ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT),
+ "for the worker offsets topic consumer when exactly-once source support is enabled",
+ false
+ );
+ }
Map<String, Object> adminProps = new HashMap<>(originals);
ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId);
@@ -111,27 +206,36 @@ public class KafkaOffsetBackingStore implements OffsetBackingStore {
adminSupplier = topicAdminSupplier;
} else {
// Create our own topic admin supplier that we'll close when we're stopped
- ownTopicAdmin = new SharedTopicAdmin(adminProps);
+ this.ownTopicAdmin = new SharedTopicAdmin(adminProps);
adminSupplier = ownTopicAdmin;
}
+ NewTopic topicDescription = newTopicDescription(topic, config);
+
+ this.offsetLog = createKafkaBasedLog(topic, producerProps, consumerProps, consumedCallback, topicDescription, adminSupplier);
+ }
+
+ private KafkaBasedLog<byte[], byte[]> createKafkaBasedLog(String topic, Map<String, Object> producerProps,
+ Map<String, Object> consumerProps,
+ Callback<ConsumerRecord<byte[], byte[]>> consumedCallback,
+ final NewTopic topicDescription, Supplier<TopicAdmin> adminSupplier) {
+ java.util.function.Consumer<TopicAdmin> createTopics = initialize(topic, topicDescription);
+ return new KafkaBasedLog<>(topic, producerProps, consumerProps, adminSupplier, consumedCallback, Time.SYSTEM, createTopics);
+ }
+
+ protected NewTopic newTopicDescription(final String topic, final WorkerConfig config) {
Map<String, Object> topicSettings = config instanceof DistributedConfig
- ? ((DistributedConfig) config).offsetStorageTopicSettings()
- : Collections.emptyMap();
- NewTopic topicDescription = TopicAdmin.defineTopic(topic)
+ ? ((DistributedConfig) config).offsetStorageTopicSettings()
+ : Collections.emptyMap();
+ return TopicAdmin.defineTopic(topic)
.config(topicSettings) // first so that we override user-supplied settings as needed
.compacted()
.partitions(config.getInt(DistributedConfig.OFFSET_STORAGE_PARTITIONS_CONFIG))
.replicationFactor(config.getShort(DistributedConfig.OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG))
.build();
-
- offsetLog = createKafkaBasedLog(topic, producerProps, consumerProps, consumedCallback, topicDescription, adminSupplier);
}
- private KafkaBasedLog<byte[], byte[]> createKafkaBasedLog(String topic, Map<String, Object> producerProps,
- Map<String, Object> consumerProps,
- Callback<ConsumerRecord<byte[], byte[]>> consumedCallback,
- final NewTopic topicDescription, Supplier<TopicAdmin> adminSupplier) {
- java.util.function.Consumer<TopicAdmin> createTopics = admin -> {
+ protected java.util.function.Consumer<TopicAdmin> initialize(final String topic, final NewTopic topicDescription) {
+ return admin -> {
log.debug("Creating admin client to manage Connect internal offset topic");
// Create the topic if it doesn't exist
Set<String> newTopics = admin.createTopics(topicDescription);
@@ -142,16 +246,40 @@ public class KafkaOffsetBackingStore implements OffsetBackingStore {
DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, "source connector offsets");
}
};
- return new KafkaBasedLog<>(topic, producerProps, consumerProps, adminSupplier, consumedCallback, Time.SYSTEM, createTopics);
}
@Override
public void start() {
log.info("Starting KafkaOffsetBackingStore");
- offsetLog.start();
+ try {
+ offsetLog.start();
+ } catch (UnsupportedVersionException e) {
+ String message;
+ if (exactlyOnce) {
+ message = "Enabling exactly-once support for source connectors requires a Kafka broker version that allows "
+ + "admin clients to read consumer offsets. Please either disable the worker's exactly-once "
+ + "support for source connectors, or upgrade to a newer Kafka broker version.";
+ } else {
+ message = "When " + ConsumerConfig.ISOLATION_LEVEL_CONFIG + "is set to "
+ + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT)
+ + ", a Kafka broker version that allows admin clients to read consumer offsets is required. "
+ + "Please either reconfigure the worker or connector, or upgrade to a newer Kafka broker version.";
+ }
+ throw new ConnectException(message, e);
+ }
log.info("Finished reading offsets topic and starting KafkaOffsetBackingStore");
}
+ /**
+ * Stop reading from and writing to the offsets topic, and relinquish resources allocated for interacting
+ * with it, including Kafka clients.
+ * <p>
+ * <b>Note:</b> if the now-deprecated {@link #KafkaOffsetBackingStore()} constructor was used to create
+ * this store, the underlying admin client allocated for interacting with the offsets topic will be closed.
+ * On the other hand, if the recommended {@link #KafkaOffsetBackingStore(Supplier)} constructor was used to
+ * create this store, the admin client derived from the given {@link Supplier} will not be closed and it is the
+ * caller's responsibility to manage its lifecycle accordingly.
+ */
@Override
public void stop() {
log.info("Stopping KafkaOffsetBackingStore");
@@ -197,7 +325,7 @@ public class KafkaOffsetBackingStore implements OffsetBackingStore {
return producerCallback;
}
- private final Callback<ConsumerRecord<byte[], byte[]>> consumedCallback = new Callback<ConsumerRecord<byte[], byte[]>>() {
+ protected final Callback<ConsumerRecord<byte[], byte[]>> consumedCallback = new Callback<ConsumerRecord<byte[], byte[]>>() {
@Override
public void onCompletion(Throwable error, ConsumerRecord<byte[], byte[]> record) {
ByteBuffer key = record.key() != null ? ByteBuffer.wrap(record.key()) : null;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
index 735c61919d..5da5be04a5 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java
@@ -25,6 +25,7 @@ import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
@@ -33,6 +34,7 @@ import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kafka.common.errors.UnsupportedVersionException;
import org.apache.kafka.common.errors.WakeupException;
import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.connect.errors.ConnectException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,10 +42,13 @@ import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Future;
@@ -88,8 +93,9 @@ public class KafkaBasedLog<K, V> {
private final Map<String, Object> consumerConfigs;
private final Callback<ConsumerRecord<K, V>> consumedCallback;
private final Supplier<TopicAdmin> topicAdminSupplier;
+ private final boolean requireAdminForOffsets;
private Consumer<K, V> consumer;
- private Producer<K, V> producer;
+ private Optional<Producer<K, V>> producer;
private TopicAdmin admin;
private Thread thread;
@@ -160,6 +166,56 @@ public class KafkaBasedLog<K, V> {
this.readLogEndOffsetCallbacks = new ArrayDeque<>();
this.time = time;
this.initializer = initializer != null ? initializer : admin -> { };
+
+ // If the consumer is configured with isolation.level = read_committed, then its end offsets method cannot be relied on
+ // as it will not take records from currently-open transactions into account. We want to err on the side of caution in that
+ // case: when users request a read to the end of the log, we will read up to the point where the latest offsets visible to the
+ // consumer are at least as high as the (possibly-part-of-a-transaction) end offsets of the topic.
+ this.requireAdminForOffsets = IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT)
+ .equals(consumerConfigs.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG));
+ }
+
+ /**
+ * Create a new KafkaBasedLog object using pre-existing Kafka clients. This does not start reading the log and writing
+ * is not permitted until {@link #start()} is invoked. Note that the consumer and (if not null) producer given to this log
+ * will be closed when this log is {@link #stop() stopped}.
+ *
+ * @param topic the topic to treat as a log
+ * @param consumer the consumer to use for reading from the log; may not be null
+ * @param producer the producer to use for writing to the log; may be null, which will create a read-only log
+ * @param topicAdmin an admin client, the lifecycle of which is expected to be controlled by the calling component;
+ * may not be null
+ * @param consumedCallback callback to invoke for each {@link ConsumerRecord} consumed when tailing the log
+ * @param time Time interface
+ * @param initializer the function that should be run when this log is {@link #start() started}; may be null
+ * @return a {@link KafkaBasedLog} using the given clients
+ */
+ public static <K, V> KafkaBasedLog<K, V> withExistingClients(String topic,
+ Consumer<K, V> consumer,
+ Producer<K, V> producer,
+ TopicAdmin topicAdmin,
+ Callback<ConsumerRecord<K, V>> consumedCallback,
+ Time time,
+ java.util.function.Consumer<TopicAdmin> initializer) {
+ Objects.requireNonNull(topicAdmin);
+ return new KafkaBasedLog<K, V>(topic,
+ Collections.emptyMap(),
+ Collections.emptyMap(),
+ () -> topicAdmin,
+ consumedCallback,
+ time,
+ initializer) {
+
+ @Override
+ protected Producer<K, V> createProducer() {
+ return producer;
+ }
+
+ @Override
+ protected Consumer<K, V> createConsumer() {
+ return consumer;
+ }
+ };
}
public void start() {
@@ -167,10 +223,19 @@ public class KafkaBasedLog<K, V> {
// Create the topic admin client and initialize the topic ...
admin = topicAdminSupplier.get(); // may be null
+ if (admin == null && requireAdminForOffsets) {
+ throw new ConnectException(
+ "Must provide a TopicAdmin to KafkaBasedLog when consumer is configured with "
+ + ConsumerConfig.ISOLATION_LEVEL_CONFIG + " set to "
+ + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT)
+ );
+ }
initializer.accept(admin);
// Then create the producer and consumer
- producer = createProducer();
+ producer = Optional.ofNullable(createProducer());
+ if (!producer.isPresent())
+ log.trace("Creating read-only KafkaBasedLog for topic " + topic);
consumer = createConsumer();
List<TopicPartition> partitions = new ArrayList<>();
@@ -214,26 +279,21 @@ public class KafkaBasedLog<K, V> {
synchronized (this) {
stopRequested = true;
}
- consumer.wakeup();
-
- try {
- thread.join();
- } catch (InterruptedException e) {
- throw new ConnectException("Failed to stop KafkaBasedLog. Exiting without cleanly shutting " +
- "down it's producer and consumer.", e);
+ if (consumer != null) {
+ consumer.wakeup();
}
- try {
- producer.close();
- } catch (KafkaException e) {
- log.error("Failed to stop KafkaBasedLog producer", e);
+ if (thread != null) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ throw new ConnectException("Failed to stop KafkaBasedLog. Exiting without cleanly shutting " +
+ "down it's producer and consumer.", e);
+ }
}
- try {
- consumer.close();
- } catch (KafkaException e) {
- log.error("Failed to stop KafkaBasedLog consumer", e);
- }
+ producer.ifPresent(p -> Utils.closeQuietly(p, "KafkaBasedLog producer for topic " + topic));
+ Utils.closeQuietly(consumer, "KafkaBasedLog consumer for topic " + topic);
// do not close the admin client, since we don't own it
admin = null;
@@ -243,7 +303,7 @@ public class KafkaBasedLog<K, V> {
/**
* Flushes any outstanding writes and then reads to the current end of the log and invokes the specified callback.
- * Note that this checks the current, offsets, reads to them, and invokes the callback regardless of whether
+ * Note that this checks the current offsets, reads to them, and invokes the callback regardless of whether
* additional records have been written to the log. If the caller needs to ensure they have truly reached the end
* of the log, they must ensure there are no other writers during this period.
*
@@ -256,7 +316,7 @@ public class KafkaBasedLog<K, V> {
*/
public void readToEnd(Callback<Void> callback) {
log.trace("Starting read to end log for topic {}", topic);
- producer.flush();
+ flush();
synchronized (this) {
readLogEndOffsetCallbacks.add(callback);
}
@@ -267,7 +327,7 @@ public class KafkaBasedLog<K, V> {
* Flush the underlying producer to ensure that all pending writes have been sent.
*/
public void flush() {
- producer.flush();
+ producer.ifPresent(Producer::flush);
}
/**
@@ -285,14 +345,16 @@ public class KafkaBasedLog<K, V> {
}
public void send(K key, V value, org.apache.kafka.clients.producer.Callback callback) {
- producer.send(new ProducerRecord<>(topic, key, value), callback);
+ producer.orElseThrow(() ->
+ new IllegalStateException("This KafkaBasedLog was created in read-only mode and does not support write operations")
+ ).send(new ProducerRecord<>(topic, key, value), callback);
}
public int partitionCount() {
return partitionCount;
}
- private Producer<K, V> createProducer() {
+ protected Producer<K, V> createProducer() {
// Always require producer acks to all to ensure durable writes
producerConfigs.put(ProducerConfig.ACKS_CONFIG, "all");
@@ -301,7 +363,7 @@ public class KafkaBasedLog<K, V> {
return new KafkaProducer<>(producerConfigs);
}
- private Consumer<K, V> createConsumer() {
+ protected Consumer<K, V> createConsumer() {
// Always force reset to the beginning of the log since this class wants to consume all available log data
consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
@@ -356,7 +418,15 @@ public class KafkaBasedLog<K, V> {
}
// Visible for testing
- Map<TopicPartition, Long> readEndOffsets(Set<TopicPartition> assignment, boolean shouldRetry) {
+ /**
+ * Read to the end of the given list of topic partitions
+ * @param assignment the topic partitions to read to the end of
+ * @param shouldRetry boolean flag to enable retry for the admin client {@code listOffsets()} call.
+ * @throws UnsupportedVersionException if the log's consumer is using the "read_committed" isolation level (and
+ * therefore a separate admin client is required to read end offsets for the topic), but the broker does not support
+ * reading end offsets using an admin client
+ */
+ Map<TopicPartition, Long> readEndOffsets(Set<TopicPartition> assignment, boolean shouldRetry) throws UnsupportedVersionException {
log.trace("Reading to end of offset log");
// Note that we'd prefer to not use the consumer to find the end offsets for the assigned topic partitions.
@@ -381,6 +451,10 @@ public class KafkaBasedLog<K, V> {
} catch (UnsupportedVersionException e) {
// This may happen with really old brokers that don't support the auto topic creation
// field in metadata requests
+ if (requireAdminForOffsets) {
+ // Should be handled by the caller during log startup
+ throw e;
+ }
log.debug("Reading to end of log offsets with consumer since admin client is unsupported: {}", e.getMessage());
// Forget the reference to the admin so that we won't even try to use the admin the next time this method is called
admin = null;
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java
index d97533a87e..f9defc77ca 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java
@@ -23,11 +23,13 @@ import org.apache.kafka.clients.admin.ConfigEntry;
import org.apache.kafka.clients.admin.CreateTopicsOptions;
import org.apache.kafka.clients.admin.DescribeConfigsOptions;
import org.apache.kafka.clients.admin.DescribeTopicsOptions;
+import org.apache.kafka.clients.admin.ListOffsetsOptions;
import org.apache.kafka.clients.admin.ListOffsetsResult;
import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.admin.OffsetSpec;
import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.config.ConfigException;
@@ -264,39 +266,34 @@ public class TopicAdmin implements AutoCloseable {
}
private static final Logger log = LoggerFactory.getLogger(TopicAdmin.class);
- private final Map<String, Object> adminConfig;
+ private final String bootstrapServers;
private final Admin admin;
private final boolean logCreation;
/**
* Create a new topic admin component with the given configuration.
+ * <p>
+ * Note that this will create an underlying {@link Admin} instance which must be freed when this
+ * topic admin is no longer needed by calling {@link #close()} or {@link #close(Duration)}.
*
* @param adminConfig the configuration for the {@link Admin}
*/
public TopicAdmin(Map<String, Object> adminConfig) {
- this(adminConfig, Admin.create(adminConfig));
+ this(adminConfig.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG), Admin.create(adminConfig));
}
// visible for testing
- TopicAdmin(Map<String, Object> adminConfig, Admin adminClient) {
- this(adminConfig, adminClient, true);
+ TopicAdmin(Object bootstrapServers, Admin adminClient) {
+ this(bootstrapServers, adminClient, true);
}
// visible for testing
- TopicAdmin(Map<String, Object> adminConfig, Admin adminClient, boolean logCreation) {
+ TopicAdmin(Object bootstrapServers, Admin adminClient, boolean logCreation) {
this.admin = adminClient;
- this.adminConfig = adminConfig != null ? adminConfig : Collections.emptyMap();
+ this.bootstrapServers = bootstrapServers != null ? bootstrapServers.toString() : "<unknown>";
this.logCreation = logCreation;
}
- /**
- * Get the {@link Admin} client used by this topic admin object.
- * @return the Kafka admin instance; never null
- */
- public Admin admin() {
- return admin;
- }
-
/**
* Attempt to create the topic described by the given definition, returning true if the topic was created or false
* if the topic already existed.
@@ -371,7 +368,6 @@ public class TopicAdmin implements AutoCloseable {
}
}
if (topicsByName.isEmpty()) return EMPTY_CREATION;
- String bootstrapServers = bootstrapServers();
String topicNameList = Utils.join(topicsByName.keySet(), "', '");
// Attempt to create any missing topics
@@ -448,7 +444,6 @@ public class TopicAdmin implements AutoCloseable {
if (topics == null) {
return Collections.emptyMap();
}
- String bootstrapServers = bootstrapServers();
String topicNameList = String.join(", ", topics);
Map<String, KafkaFuture<TopicDescription>> newResults =
@@ -604,7 +599,6 @@ public class TopicAdmin implements AutoCloseable {
if (topics.isEmpty()) {
return Collections.emptyMap();
}
- String bootstrapServers = bootstrapServers();
String topicNameList = String.join(", ", topics);
Collection<ConfigResource> resources = topics.stream()
.map(t -> new ConfigResource(ConfigResource.Type.TOPIC, t))
@@ -664,7 +658,7 @@ public class TopicAdmin implements AutoCloseable {
return Collections.emptyMap();
}
Map<TopicPartition, OffsetSpec> offsetSpecMap = partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> OffsetSpec.latest()));
- ListOffsetsResult resultFuture = admin.listOffsets(offsetSpecMap);
+ ListOffsetsResult resultFuture = admin.listOffsets(offsetSpecMap, new ListOffsetsOptions(IsolationLevel.READ_UNCOMMITTED));
// Get the individual result for each topic partition so we have better error messages
Map<TopicPartition, Long> result = new HashMap<>();
for (TopicPartition partition : partitions) {
@@ -675,28 +669,28 @@ public class TopicAdmin implements AutoCloseable {
Throwable cause = e.getCause();
String topic = partition.topic();
if (cause instanceof AuthorizationException) {
- String msg = String.format("Not authorized to get the end offsets for topic '%s' on brokers at %s", topic, bootstrapServers());
+ String msg = String.format("Not authorized to get the end offsets for topic '%s' on brokers at %s", topic, bootstrapServers);
throw new ConnectException(msg, e);
} else if (cause instanceof UnsupportedVersionException) {
// Should theoretically never happen, because this method is the same as what the consumer uses and therefore
// should exist in the broker since before the admin client was added
- String msg = String.format("API to get the get the end offsets for topic '%s' is unsupported on brokers at %s", topic, bootstrapServers());
+ String msg = String.format("API to get the get the end offsets for topic '%s' is unsupported on brokers at %s", topic, bootstrapServers);
throw new UnsupportedVersionException(msg, e);
} else if (cause instanceof TimeoutException) {
- String msg = String.format("Timed out while waiting to get end offsets for topic '%s' on brokers at %s", topic, bootstrapServers());
+ String msg = String.format("Timed out while waiting to get end offsets for topic '%s' on brokers at %s", topic, bootstrapServers);
throw new TimeoutException(msg, e);
} else if (cause instanceof LeaderNotAvailableException) {
- String msg = String.format("Unable to get end offsets during leader election for topic '%s' on brokers at %s", topic, bootstrapServers());
+ String msg = String.format("Unable to get end offsets during leader election for topic '%s' on brokers at %s", topic, bootstrapServers);
throw new LeaderNotAvailableException(msg, e);
} else if (cause instanceof org.apache.kafka.common.errors.RetriableException) {
throw (org.apache.kafka.common.errors.RetriableException) cause;
} else {
- String msg = String.format("Error while getting end offsets for topic '%s' on brokers at %s", topic, bootstrapServers());
+ String msg = String.format("Error while getting end offsets for topic '%s' on brokers at %s", topic, bootstrapServers);
throw new ConnectException(msg, e);
}
} catch (InterruptedException e) {
Thread.interrupted();
- String msg = String.format("Interrupted while attempting to read end offsets for topic '%s' on brokers at %s", partition.topic(), bootstrapServers());
+ String msg = String.format("Interrupted while attempting to read end offsets for topic '%s' on brokers at %s", partition.topic(), bootstrapServers);
throw new RetriableException(msg, e);
}
}
@@ -742,9 +736,4 @@ public class TopicAdmin implements AutoCloseable {
public void close(Duration timeout) {
admin.close(timeout);
}
-
- private String bootstrapServers() {
- Object servers = adminConfig.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG);
- return servers != null ? servers.toString() : "<unknown>";
- }
}
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
index be26149d27..d4184bbab2 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java
@@ -381,9 +381,10 @@ public class ErrorHandlingTaskTest {
EasyMock.expect(workerSourceTask.commitOffsets()).andReturn(true);
+ offsetStore.start();
+ EasyMock.expectLastCall();
sourceTask.initialize(EasyMock.anyObject());
EasyMock.expectLastCall();
-
sourceTask.start(EasyMock.anyObject());
EasyMock.expectLastCall();
@@ -445,9 +446,10 @@ public class ErrorHandlingTaskTest {
EasyMock.expect(workerSourceTask.commitOffsets()).andReturn(true);
+ offsetStore.start();
+ EasyMock.expectLastCall();
sourceTask.initialize(EasyMock.anyObject());
EasyMock.expectLastCall();
-
sourceTask.start(EasyMock.anyObject());
EasyMock.expectLastCall();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java
index 5cf34cb911..fdc253d04e 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java
@@ -351,6 +351,9 @@ public class ExactlyOnceWorkerSourceTaskTest extends ThreadedTest {
createWorkerTask();
expectCall(preProducerCheck::run);
+ expectCall(producer::initTransactions);
+ expectCall(postProducerCheck::run);
+
Exception exception = new ConnectException("No soup for you!");
expectCall(offsetStore::start).andThrow(exception);
@@ -374,7 +377,6 @@ public class ExactlyOnceWorkerSourceTaskTest extends ThreadedTest {
createWorkerTask();
expectCall(preProducerCheck::run);
- expectCall(offsetStore::start);
expectCall(producer::initTransactions);
Exception exception = new ConnectException("You can't do that!");
expectCall(postProducerCheck::run).andThrow(exception);
@@ -399,7 +401,6 @@ public class ExactlyOnceWorkerSourceTaskTest extends ThreadedTest {
createWorkerTask();
expectCall(preProducerCheck::run);
- expectCall(offsetStore::start);
Exception exception = new ConnectException("New task configs for the connector have already been generated");
expectCall(producer::initTransactions).andThrow(exception);
@@ -1293,9 +1294,9 @@ public class ExactlyOnceWorkerSourceTaskTest extends ThreadedTest {
private void expectPreflight() {
expectCall(preProducerCheck::run);
- expectCall(offsetStore::start);
expectCall(producer::initTransactions);
expectCall(postProducerCheck::run);
+ expectCall(offsetStore::start);
}
private void expectStartup() {
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java
index 253efff8c0..f7c18d74a2 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java
@@ -89,6 +89,9 @@ public class WorkerConnectorTest extends EasyMockSupport {
connector.version();
expectLastCall().andReturn(VERSION);
+ offsetStore.start();
+ expectLastCall();
+
connector.initialize(EasyMock.notNull(SourceConnectorContext.class));
expectLastCall().andThrow(exception);
@@ -173,6 +176,9 @@ public class WorkerConnectorTest extends EasyMockSupport {
connector.version();
expectLastCall().andReturn(VERSION);
+ offsetStore.start();
+ expectLastCall();
+
connector.initialize(EasyMock.notNull(SourceConnectorContext.class));
expectLastCall();
@@ -281,6 +287,9 @@ public class WorkerConnectorTest extends EasyMockSupport {
connector.initialize(EasyMock.notNull(SourceConnectorContext.class));
expectLastCall();
+ offsetStore.start();
+ expectLastCall();
+
listener.onPause(CONNECTOR);
expectLastCall();
@@ -429,6 +438,9 @@ public class WorkerConnectorTest extends EasyMockSupport {
connector.version();
expectLastCall().andReturn(VERSION);
+ offsetStore.start();
+ expectLastCall();
+
connector.initialize(EasyMock.notNull(SourceConnectorContext.class));
expectLastCall();
@@ -478,6 +490,9 @@ public class WorkerConnectorTest extends EasyMockSupport {
connector.version();
expectLastCall().andReturn(VERSION);
+ offsetStore.start();
+ expectLastCall();
+
connector.initialize(EasyMock.notNull(SourceConnectorContext.class));
expectLastCall();
@@ -530,6 +545,9 @@ public class WorkerConnectorTest extends EasyMockSupport {
connector.version();
expectLastCall().andReturn(VERSION);
+ offsetStore.start();
+ expectLastCall();
+
connector.initialize(EasyMock.notNull(SourceConnectorContext.class));
expectLastCall();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
index 322c37f05d..5ce0e44f3e 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java
@@ -269,12 +269,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
public void testPause() throws Exception {
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
AtomicInteger count = new AtomicInteger(0);
CountDownLatch pollLatch = expectPolls(10, count);
@@ -323,12 +318,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
public void testPollsInBackground() throws Exception {
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
final CountDownLatch pollLatch = expectPolls(10);
// In this test, we don't flush, so nothing goes any further than the offset writer
@@ -366,12 +356,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
public void testFailureInPoll() throws Exception {
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
final CountDownLatch pollLatch = new CountDownLatch(1);
final RuntimeException exception = new RuntimeException();
@@ -408,12 +393,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
public void testFailureInPollAfterCancel() throws Exception {
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
final CountDownLatch pollLatch = new CountDownLatch(1);
final CountDownLatch workerCancelLatch = new CountDownLatch(1);
@@ -456,12 +436,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
public void testFailureInPollAfterStop() throws Exception {
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
final CountDownLatch pollLatch = new CountDownLatch(1);
final CountDownLatch workerStopLatch = new CountDownLatch(1);
@@ -502,12 +477,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
// Test that the task handles an empty list of records
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
// We'll wait for some data, then trigger a flush
final CountDownLatch pollLatch = expectEmptyPolls(1, new AtomicInteger());
@@ -543,12 +513,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
// Test that the task commits properly when prompted
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
// We'll wait for some data, then trigger a flush
final CountDownLatch pollLatch = expectPolls(1);
@@ -589,12 +554,7 @@ public class WorkerSourceTaskTest extends ThreadedTest {
// Test that the task commits properly when prompted
createWorkerTask();
- sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
- EasyMock.expectLastCall();
- sourceTask.start(TASK_PROPS);
- EasyMock.expectLastCall();
- statusListener.onStartup(taskId);
- EasyMock.expectLastCall();
+ expectCleanStartup();
// We'll wait for some data, then trigger a flush
final CountDownLatch pollLatch = expectPolls(1);
@@ -763,6 +723,8 @@ public class WorkerSourceTaskTest extends ThreadedTest {
createWorkerTask();
+ offsetStore.start();
+ EasyMock.expectLastCall();
sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
EasyMock.expectLastCall();
sourceTask.start(TASK_PROPS);
@@ -1088,6 +1050,17 @@ public class WorkerSourceTaskTest extends ThreadedTest {
private abstract static class TestSourceTask extends SourceTask {
}
+ private void expectCleanStartup() {
+ offsetStore.start();
+ EasyMock.expectLastCall();
+ sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class));
+ EasyMock.expectLastCall();
+ sourceTask.start(TASK_PROPS);
+ EasyMock.expectLastCall();
+ statusListener.onStartup(taskId);
+ EasyMock.expectLastCall();
+ }
+
private void expectClose() {
producer.close(EasyMock.anyObject(Duration.class));
EasyMock.expectLastCall();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
index a15cf77e32..a064e296b2 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java
@@ -16,11 +16,11 @@
*/
package org.apache.kafka.connect.runtime;
-import java.util.Collection;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.admin.Admin;
import org.apache.kafka.clients.admin.FenceProducersResult;
import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.MetricName;
@@ -54,6 +54,7 @@ import org.apache.kafka.connect.sink.SinkTask;
import org.apache.kafka.connect.source.SourceConnector;
import org.apache.kafka.connect.source.SourceRecord;
import org.apache.kafka.connect.source.SourceTask;
+import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore;
import org.apache.kafka.connect.storage.Converter;
import org.apache.kafka.connect.storage.HeaderConverter;
import org.apache.kafka.connect.storage.OffsetBackingStore;
@@ -63,6 +64,7 @@ import org.apache.kafka.connect.util.ConnectorTaskId;
import org.apache.kafka.connect.util.FutureCallback;
import org.apache.kafka.connect.util.ParameterizedTest;
import org.apache.kafka.connect.util.ThreadedTest;
+import org.apache.kafka.connect.util.TopicAdmin;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -80,6 +82,7 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.lang.management.ManagementFactory;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -96,6 +99,11 @@ import java.util.function.Function;
import static org.apache.kafka.clients.admin.AdminClientConfig.RETRY_BACKOFF_MS_CONFIG;
import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX;
import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG;
+
+import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG;
+import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG;
+import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG;
+import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX;
import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX;
import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG;
import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG;
@@ -114,6 +122,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -243,7 +252,7 @@ public class WorkerTest extends ThreadedTest {
defaultProducerConfigs.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1");
defaultProducerConfigs.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE));
- defaultConsumerConfigs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
+ defaultConsumerConfigs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
defaultConsumerConfigs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
defaultConsumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
defaultConsumerConfigs
@@ -488,6 +497,7 @@ public class WorkerTest extends ThreadedTest {
pluginsMockedStatic.when(() -> Plugins.compareAndSwapLoaders(pluginLoader)).thenReturn(delegatingLoader);
pluginsMockedStatic.when(() -> Plugins.compareAndSwapLoaders(delegatingLoader)).thenReturn(pluginLoader);
connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, shortConnectorAlias);
+
connectorProps.put(SinkConnectorConfig.TOPICS_CONFIG, "gfieyls, wfru");
worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, noneConnectorClientConfigOverridePolicy);
@@ -1269,7 +1279,6 @@ public class WorkerTest extends ThreadedTest {
props.put("admin.client.id", "testid");
props.put("admin.metadata.max.age.ms", "5000");
WorkerConfig configWithOverrides = new StandaloneConfig(props);
-
Map<String, Object> connConfig = Collections.singletonMap("metadata.max.age.ms", "10000");
when(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX)).thenReturn(connConfig);
@@ -1279,6 +1288,463 @@ public class WorkerTest extends ThreadedTest {
verify(connectorConfig).originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX);
}
+ @Test
+ public void testRegularSourceOffsetsConsumerConfigs() {
+ final Map<String, Object> connectorConsumerOverrides = new HashMap<>();
+ when(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)).thenReturn(connectorConsumerOverrides);
+
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "enabled");
+ workerProps.put("bootstrap.servers", "localhost:4761");
+ workerProps.put("group.id", "connect-cluster");
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", "connect-offsets");
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ Map<String, Object> consumerConfigs = Worker.regularSourceOffsetsConsumerConfigs(
+ "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:4761", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG));
+
+ workerProps.put("consumer." + BOOTSTRAP_SERVERS_CONFIG, "localhost:9021");
+ workerProps.put("consumer." + ISOLATION_LEVEL_CONFIG, "read_uncommitted");
+ config = new DistributedConfig(workerProps);
+ consumerConfigs = Worker.regularSourceOffsetsConsumerConfigs(
+ "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:9021", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ // User is allowed to override the isolation level for regular (non-exactly-once) source connectors and their tasks
+ assertEquals("read_uncommitted", consumerConfigs.get(ISOLATION_LEVEL_CONFIG));
+
+ workerProps.remove("consumer." + ISOLATION_LEVEL_CONFIG);
+ connectorConsumerOverrides.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:489");
+ connectorConsumerOverrides.put(ISOLATION_LEVEL_CONFIG, "read_uncommitted");
+ config = new DistributedConfig(workerProps);
+ consumerConfigs = Worker.regularSourceOffsetsConsumerConfigs(
+ "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:489", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ // User is allowed to override the isolation level for regular (non-exactly-once) source connectors and their tasks
+ assertEquals("read_uncommitted", consumerConfigs.get(ISOLATION_LEVEL_CONFIG));
+ }
+
+ @Test
+ public void testExactlyOnceSourceOffsetsConsumerConfigs() {
+ final Map<String, Object> connectorConsumerOverrides = new HashMap<>();
+ when(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)).thenReturn(connectorConsumerOverrides);
+
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "enabled");
+ workerProps.put("bootstrap.servers", "localhost:4761");
+ workerProps.put("group.id", "connect-cluster");
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", "connect-offsets");
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ Map<String, Object> consumerConfigs = Worker.exactlyOnceSourceOffsetsConsumerConfigs(
+ "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:4761", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG));
+
+ workerProps.put("consumer." + BOOTSTRAP_SERVERS_CONFIG, "localhost:9021");
+ workerProps.put("consumer." + ISOLATION_LEVEL_CONFIG, "read_uncommitted");
+ config = new DistributedConfig(workerProps);
+ consumerConfigs = Worker.exactlyOnceSourceOffsetsConsumerConfigs(
+ "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:9021", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ // User is not allowed to override isolation level when exactly-once support is enabled
+ assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG));
+
+ workerProps.remove("consumer." + ISOLATION_LEVEL_CONFIG);
+ connectorConsumerOverrides.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:489");
+ connectorConsumerOverrides.put(ISOLATION_LEVEL_CONFIG, "read_uncommitted");
+ config = new DistributedConfig(workerProps);
+ consumerConfigs = Worker.exactlyOnceSourceOffsetsConsumerConfigs(
+ "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:489", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ // User is not allowed to override isolation level when exactly-once support is enabled
+ assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG));
+ }
+
+ @Test
+ public void testExactlyOnceSourceTaskProducerConfigs() {
+ final Map<String, Object> connectorProducerOverrides = new HashMap<>();
+ when(connectorConfig.originalsWithPrefix(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)).thenReturn(connectorProducerOverrides);
+
+ final String groupId = "connect-cluster";
+ final String transactionalId = Worker.taskTransactionalId(groupId, TASK_ID.connector(), TASK_ID.task());
+
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "enabled");
+ workerProps.put("bootstrap.servers", "localhost:4761");
+ workerProps.put("group.id", groupId);
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", "connect-offsets");
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ Map<String, Object> producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs(
+ TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:4761", producerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG));
+ assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG));
+
+ workerProps.put("producer." + BOOTSTRAP_SERVERS_CONFIG, "localhost:9021");
+ workerProps.put("producer." + ENABLE_IDEMPOTENCE_CONFIG, "false");
+ workerProps.put("producer." + TRANSACTIONAL_ID_CONFIG, "some-other-transactional-id");
+ config = new DistributedConfig(workerProps);
+ producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs(
+ TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:9021", producerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ // User is not allowed to override idempotence or transactional ID for exactly-once source tasks
+ assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG));
+ assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG));
+
+ workerProps.remove("producer." + ENABLE_IDEMPOTENCE_CONFIG);
+ workerProps.remove("producer." + TRANSACTIONAL_ID_CONFIG);
+ connectorProducerOverrides.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:489");
+ connectorProducerOverrides.put(ENABLE_IDEMPOTENCE_CONFIG, "false");
+ connectorProducerOverrides.put(TRANSACTIONAL_ID_CONFIG, "yet-another-transactional-id");
+ config = new DistributedConfig(workerProps);
+ producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs(
+ TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ assertEquals("localhost:489", producerConfigs.get(BOOTSTRAP_SERVERS_CONFIG));
+ // User is not allowed to override idempotence or transactional ID for exactly-once source tasks
+ assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG));
+ assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG));
+
+ // Rare case: somehow, an explicit null has made it into the connector config
+ connectorProducerOverrides.put(TRANSACTIONAL_ID_CONFIG, null);
+ producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs(
+ TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID);
+ // User is still not allowed to override idempotence or transactional ID for exactly-once source tasks
+ assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG));
+ assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG));
+ }
+
+ @Test
+ public void testOffsetStoreForRegularSourceConnector() {
+ mockInternalConverters();
+ mockFileConfigProvider();
+
+ final String workerOffsetsTopic = "worker-offsets";
+ final String workerBootstrapServers = "localhost:4761";
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "disabled");
+ workerProps.put("bootstrap.servers", workerBootstrapServers);
+ workerProps.put("group.id", "connect-cluster");
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", workerOffsetsTopic);
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy);
+ worker.start();
+
+ Map<String, String> connectorProps = new HashMap<>();
+ connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
+ connectorProps.put(CONNECTOR_CLASS_CONFIG, SampleSourceConnector.class.getName());
+ connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
+ SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With no connector-specific offsets topic in the config, we should only use the worker-global offsets store
+ ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertFalse(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic");
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a
+ // connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers
+ // for the connector, we should only use a connector-specific offsets store
+ connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that exactly matches the worker's, we should only use a connector-specific offsets store
+ connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, "localhost:1111");
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With no connector-specific offsets topic in the config, even with an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should still only use the worker-global offsets store
+ connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertFalse(connectorStore.hasConnectorSpecificStore());
+
+ worker.stop();
+ }
+
+ @Test
+ public void testOffsetStoreForExactlyOnceSourceConnector() {
+ mockInternalConverters();
+ mockFileConfigProvider();
+
+ final String workerOffsetsTopic = "worker-offsets";
+ final String workerBootstrapServers = "localhost:4761";
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "enabled");
+ workerProps.put("bootstrap.servers", workerBootstrapServers);
+ workerProps.put("group.id", "connect-cluster");
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", workerOffsetsTopic);
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy);
+ worker.start();
+
+ Map<String, String> connectorProps = new HashMap<>();
+ connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
+ connectorProps.put(CONNECTOR_CLASS_CONFIG, SampleSourceConnector.class.getName());
+ connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
+ SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With no connector-specific offsets topic in the config, we should only use a connector-specific offsets store
+ ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic");
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a
+ // connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers
+ // for the connector, we should only use a connector-specific store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that exactly matches the worker's, we should only use a connector-specific store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, "localhost:1111");
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With no connector-specific offsets topic in the config and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ worker.stop();
+ }
+
+ @Test
+ public void testOffsetStoreForRegularSourceTask() {
+ mockInternalConverters();
+ mockFileConfigProvider();
+
+ Map<String, Object> producerProps = new HashMap<>();
+ @SuppressWarnings("unchecked")
+ Producer<byte[], byte[]> producer = mock(Producer.class);
+ TopicAdmin topicAdmin = mock(TopicAdmin.class);
+
+ final String workerOffsetsTopic = "worker-offsets";
+ final String workerBootstrapServers = "localhost:4761";
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "disabled");
+ workerProps.put("bootstrap.servers", workerBootstrapServers);
+ workerProps.put("group.id", "connect-cluster");
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", workerOffsetsTopic);
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy);
+ worker.start();
+
+ Map<String, String> connectorProps = new HashMap<>();
+ connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
+ connectorProps.put(CONNECTOR_CLASS_CONFIG, SampleSourceConnector.class.getName());
+ connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
+
+ final SourceConnectorConfig sourceConfigWithoutOffsetsTopic = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers);
+ // With no connector-specific offsets topic in the config, we should only use the worker-global store
+ // Pass in a null topic admin to make sure that with these parameters, the method doesn't require a topic admin
+ ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfigWithoutOffsetsTopic, sourceConnector.getClass(), producer, producerProps, null);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertFalse(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic");
+ final SourceConnectorConfig sourceConfigWithOffsetsTopic = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a
+ // connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfigWithOffsetsTopic, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+ assertThrows(NullPointerException.class,
+ () -> worker.offsetStoreForRegularSourceTask(
+ TASK_ID, sourceConfigWithOffsetsTopic, sourceConnector.getClass(), producer, producerProps, null
+ )
+ );
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic);
+ final SourceConnectorConfig sourceConfigWithSameOffsetsTopicAsWorker = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers
+ // for the connector, we should only use a connector-specific store
+ connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfigWithSameOffsetsTopicAsWorker, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+ assertThrows(
+ NullPointerException.class,
+ () -> worker.offsetStoreForRegularSourceTask(
+ TASK_ID, sourceConfigWithSameOffsetsTopicAsWorker, sourceConnector.getClass(), producer, producerProps, null
+ )
+ );
+
+ producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that exactly matches the worker's, we should only use a connector-specific store
+ connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfigWithSameOffsetsTopicAsWorker, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+ assertThrows(
+ NullPointerException.class,
+ () -> worker.offsetStoreForRegularSourceTask(
+ TASK_ID, sourceConfigWithSameOffsetsTopicAsWorker, sourceConnector.getClass(), producer, producerProps, null
+ )
+ );
+
+ producerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:1111");
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfigWithSameOffsetsTopicAsWorker, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+ assertThrows(
+ NullPointerException.class,
+ () -> worker.offsetStoreForRegularSourceTask(
+ TASK_ID, sourceConfigWithSameOffsetsTopicAsWorker, sourceConnector.getClass(), producer, producerProps, null
+ )
+ );
+
+ connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG);
+ // With no connector-specific offsets topic in the config and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should still only use the worker-global store
+ // Pass in a null topic admin to make sure that with these parameters, the method doesn't require a topic admin
+ connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfigWithoutOffsetsTopic, sourceConnector.getClass(), producer, producerProps, null);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertFalse(connectorStore.hasConnectorSpecificStore());
+
+ worker.stop();
+ }
+
+ @Test
+ public void testOffsetStoreForExactlyOnceSourceTask() {
+ mockInternalConverters();
+ mockFileConfigProvider();
+
+ Map<String, Object> producerProps = new HashMap<>();
+ @SuppressWarnings("unchecked")
+ Producer<byte[], byte[]> producer = mock(Producer.class);
+ TopicAdmin topicAdmin = mock(TopicAdmin.class);
+
+ final String workerOffsetsTopic = "worker-offsets";
+ final String workerBootstrapServers = "localhost:4761";
+ Map<String, String> workerProps = new HashMap<>(this.workerProps);
+ workerProps.put("exactly.once.source.support", "enabled");
+ workerProps.put("bootstrap.servers", workerBootstrapServers);
+ workerProps.put("group.id", "connect-cluster");
+ workerProps.put("config.storage.topic", "connect-configs");
+ workerProps.put("offset.storage.topic", workerOffsetsTopic);
+ workerProps.put("status.storage.topic", "connect-statuses");
+ config = new DistributedConfig(workerProps);
+
+ worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy);
+ worker.start();
+
+ Map<String, String> connectorProps = new HashMap<>();
+ connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
+ connectorProps.put(CONNECTOR_CLASS_CONFIG, SampleSourceConnector.class.getName());
+ connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
+ SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers);
+ // With no connector-specific offsets topic in the config, we should only use a connector-specific offsets store
+ ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic");
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a
+ // connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers
+ // for the connector, we should only use a connector-specific store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that exactly matches the worker's, we should only use a connector-specific store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertFalse(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ producerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:1111");
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG);
+ sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation);
+ // With no connector-specific offsets topic in the config and an overridden bootstrap.servers
+ // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store
+ connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin);
+ assertTrue(connectorStore.hasWorkerGlobalStore());
+ assertTrue(connectorStore.hasConnectorSpecificStore());
+
+ worker.stop();
+ }
+
@Test
public void testWorkerMetrics() throws Exception {
mockInternalConverters();
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
index 2ab7c38a3b..cf11230f3d 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java
@@ -21,6 +21,7 @@ import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.header.internals.RecordHeaders;
import org.apache.kafka.common.record.TimestampType;
@@ -46,6 +47,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
@@ -54,6 +56,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG;
+import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -384,6 +388,87 @@ public class KafkaOffsetBackingStoreTest {
PowerMock.verifyAll();
}
+ @Test
+ public void testConsumerPropertiesInsertedByDefaultWithExactlyOnceSourceEnabled() throws Exception {
+ Map<String, String> workerProps = new HashMap<>(DEFAULT_PROPS);
+ workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled");
+ workerProps.remove(ISOLATION_LEVEL_CONFIG);
+ DistributedConfig config = new DistributedConfig(workerProps);
+
+ expectConfigure();
+ expectClusterId();
+ PowerMock.replayAll();
+
+ store.configure(config);
+
+ assertEquals(
+ IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT),
+ capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG)
+ );
+
+ PowerMock.verifyAll();
+ }
+
+ @Test
+ public void testConsumerPropertiesOverrideUserSuppliedValuesWithExactlyOnceSourceEnabled() throws Exception {
+ Map<String, String> workerProps = new HashMap<>(DEFAULT_PROPS);
+ workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled");
+ workerProps.put(ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT));
+ DistributedConfig config = new DistributedConfig(workerProps);
+
+ expectConfigure();
+ expectClusterId();
+ PowerMock.replayAll();
+
+ store.configure(config);
+
+ assertEquals(
+ IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT),
+ capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG)
+ );
+
+ PowerMock.verifyAll();
+ }
+
+ @Test
+ public void testConsumerPropertiesNotInsertedByDefaultWithoutExactlyOnceSourceEnabled() throws Exception {
+ Map<String, String> workerProps = new HashMap<>(DEFAULT_PROPS);
+ workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "disabled");
+ workerProps.remove(ISOLATION_LEVEL_CONFIG);
+ DistributedConfig config = new DistributedConfig(workerProps);
+
+ expectConfigure();
+ expectClusterId();
+ PowerMock.replayAll();
+
+ store.configure(config);
+
+ assertNull(capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG));
+
+ PowerMock.verifyAll();
+ }
+
+ @Test
+ public void testConsumerPropertiesDoNotOverrideUserSuppliedValuesWithoutExactlyOnceSourceEnabled() throws Exception {
+ Map<String, String> workerProps = new HashMap<>(DEFAULT_PROPS);
+ workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "disabled");
+ workerProps.put(ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT));
+ DistributedConfig config = new DistributedConfig(workerProps);
+
+ expectConfigure();
+ expectClusterId();
+ PowerMock.replayAll();
+
+ store.configure(config);
+
+ assertEquals(
+ IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT),
+ capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG)
+ );
+
+ PowerMock.verifyAll();
+ }
+
private void expectConfigure() throws Exception {
PowerMock.expectPrivate(store, "createKafkaBasedLog", EasyMock.capture(capturedTopic), EasyMock.capture(capturedProducerProps),
EasyMock.capture(capturedConsumerProps), EasyMock.capture(capturedConsumedCallback),