You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2020/05/25 00:36:17 UTC
[GitHub] [kafka] levzem commented on a change in pull request #8720: KAFKA-9971: Error Reporting in Sink Connectors
levzem commented on a change in pull request #8720:
URL: https://github.com/apache/kafka/pull/8720#discussion_r429689645
##########
File path: connect/api/src/main/java/org/apache/kafka/connect/sink/ErrantRecordReporter.java
##########
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.sink;
+
+import java.util.concurrent.Future;
+
+public interface ErrantRecordReporter {
+
+
+ /**
+ * Report a problematic record and the corresponding error to be written to the sink
+ * connector's dead letter queue (DLQ).
Review comment:
this is separate from the Connect DLQ - maybe we should avoid mentioning it in the docs to make sure its a different concept?
```suggestion
* Report a problematic record and the corresponding error to be written to the sink
* connector's error topic.
```
or something similar
##########
File path: connect/api/src/main/java/org/apache/kafka/connect/sink/ErrantRecordReporter.java
##########
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.sink;
+
+import java.util.concurrent.Future;
+
+public interface ErrantRecordReporter {
+
+
Review comment:
nit 2x
##########
File path: connect/api/src/main/java/org/apache/kafka/connect/sink/ErrantRecordReporter.java
##########
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.sink;
+
+import java.util.concurrent.Future;
+
+public interface ErrantRecordReporter {
+
+
+ /**
+ * Report a problematic record and the corresponding error to be written to the sink
+ * connector's dead letter queue (DLQ).
+ *
+ * <p>This call is asynchronous and returns a {@link java.util.concurrent.Future Future}.
+ * Invoking {@link java.util.concurrent.Future#get() get()} on this future will block until the
+ * record has been written or throw any exception that occurred while sending the record.
+ * If you want to simulate a simple blocking call you can call the <code>get()</code> method
+ * immediately.
+ *
+ * @param record the problematic record; may not be null
+ * @param error the error capturing the problem with the record; may not be null
+ * @return a future that can be used to block until the record and error are reported
+ * to the DLQ
Review comment:
same as above
##########
File path: connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporter.java
##########
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.header.Header;
+import org.apache.kafka.connect.header.Headers;
+import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter;
+import org.apache.kafka.connect.sink.ErrantRecordReporter;
+import org.apache.kafka.connect.sink.SinkRecord;
+
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class WorkerErrantRecordReporter implements ErrantRecordReporter {
+
+ private static final int DLQ_NUM_DESIRED_PARTITIONS = 1;
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerErrantRecordReporter.class);
+
+ private KafkaProducer<byte[], byte[]> producer;
+ private String dlqTopic;
+ private boolean useDlq;
+ private Converter keyConverter;
+ private Converter valueConverter;
+ private List<ErrantRecordFuture> errantRecordFutures;
+ private SinkConnectorConfig sinkConfig;
+ private HeaderConverter headerConverter;
+
+
+ public static WorkerErrantRecordReporter createAndSetup(
+ Map<String, Object> adminProps,
+ Map<String, Object> producerProps,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+
+ KafkaProducer<byte[], byte[]> kafkaProducer = DeadLetterQueueReporter.setUpTopicAndProducer(
+ adminProps,
+ producerProps,
+ sinkConnectorConfig,
+ DLQ_NUM_DESIRED_PARTITIONS
+ );
+
+ return new WorkerErrantRecordReporter(
+ kafkaProducer,
+ sinkConnectorConfig,
+ workerKeyConverter,
+ workerValueConverter,
+ workerHeaderConverter
+ );
+ }
+
+ // Visible for testing purposes
+ public WorkerErrantRecordReporter(
+ KafkaProducer<byte[], byte[]> kafkaProducer,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+ producer = kafkaProducer;
+ dlqTopic = sinkConnectorConfig.dlqTopicName();
+ useDlq = dlqTopic != null && !dlqTopic.isEmpty();
+ keyConverter = workerKeyConverter;
+ valueConverter = workerValueConverter;
+ errantRecordFutures = new ArrayList<>();
+ sinkConfig = sinkConnectorConfig;
+ headerConverter = workerHeaderConverter;
+ }
+
+ @Override
+ public Future<Void> report(SinkRecord record, Throwable error) {
+
+ if (sinkConfig.enableErrorLog()) {
+ if (sinkConfig.includeRecordDetailsInErrorLog()) {
+ log.error("Error processing record: " + record.toString(), error);
+ } else {
+ log.error(
+ "Error processing record in topic "
+ + record.topic()
+ + "at offset "
+ + record.kafkaOffset(),
+ error
+ );
Review comment:
```suggestion
log.error(
"Error processing record in topic {} partition {} at offset {}",
record.topic(),
record.partition(),
record.kafkaOffset(),
error
);
```
##########
File path: connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporter.java
##########
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.header.Header;
+import org.apache.kafka.connect.header.Headers;
+import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter;
+import org.apache.kafka.connect.sink.ErrantRecordReporter;
+import org.apache.kafka.connect.sink.SinkRecord;
+
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class WorkerErrantRecordReporter implements ErrantRecordReporter {
+
+ private static final int DLQ_NUM_DESIRED_PARTITIONS = 1;
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerErrantRecordReporter.class);
+
+ private KafkaProducer<byte[], byte[]> producer;
+ private String dlqTopic;
+ private boolean useDlq;
+ private Converter keyConverter;
+ private Converter valueConverter;
+ private List<ErrantRecordFuture> errantRecordFutures;
+ private SinkConnectorConfig sinkConfig;
+ private HeaderConverter headerConverter;
+
+
+ public static WorkerErrantRecordReporter createAndSetup(
+ Map<String, Object> adminProps,
+ Map<String, Object> producerProps,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+
+ KafkaProducer<byte[], byte[]> kafkaProducer = DeadLetterQueueReporter.setUpTopicAndProducer(
+ adminProps,
+ producerProps,
+ sinkConnectorConfig,
+ DLQ_NUM_DESIRED_PARTITIONS
+ );
+
+ return new WorkerErrantRecordReporter(
+ kafkaProducer,
+ sinkConnectorConfig,
+ workerKeyConverter,
+ workerValueConverter,
+ workerHeaderConverter
+ );
+ }
+
+ // Visible for testing purposes
+ public WorkerErrantRecordReporter(
+ KafkaProducer<byte[], byte[]> kafkaProducer,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+ producer = kafkaProducer;
+ dlqTopic = sinkConnectorConfig.dlqTopicName();
+ useDlq = dlqTopic != null && !dlqTopic.isEmpty();
+ keyConverter = workerKeyConverter;
+ valueConverter = workerValueConverter;
+ errantRecordFutures = new ArrayList<>();
+ sinkConfig = sinkConnectorConfig;
+ headerConverter = workerHeaderConverter;
+ }
+
+ @Override
+ public Future<Void> report(SinkRecord record, Throwable error) {
+
+ if (sinkConfig.enableErrorLog()) {
+ if (sinkConfig.includeRecordDetailsInErrorLog()) {
+ log.error("Error processing record: " + record.toString(), error);
+ } else {
+ log.error(
+ "Error processing record in topic "
+ + record.topic()
+ + "at offset "
+ + record.kafkaOffset(),
+ error
+ );
+ }
+ }
+
+ Future<RecordMetadata> producerFuture = null;
+
+ if (useDlq) {
+
+ Headers headers = record.headers();
+ RecordHeaders result = new RecordHeaders();
+ if (headers != null) {
+ String topic = record.topic();
+ for (Header header : headers) {
+ String key = header.key();
+ byte[] rawHeader = headerConverter.fromConnectHeader(topic, key, header.schema(), header.value());
+ result.add(key, rawHeader);
+ }
+ }
+
+ ProducerRecord<byte[], byte[]> errantRecord = new ProducerRecord<>(
+ dlqTopic,
+ null,
+ record.timestamp() == RecordBatch.NO_TIMESTAMP ? record.timestamp() : null,
+ keyConverter.fromConnectData(dlqTopic, record.keySchema(), record.key()),
+ valueConverter.fromConnectData(dlqTopic, record.valueSchema(), record.value()),
+ result
+ );
+
+ producerFuture = producer.send(errantRecord);
+ }
+
+ ErrantRecordFuture errantRecordFuture = new ErrantRecordFuture(producerFuture);
+ errantRecordFutures.add(errantRecordFuture);
+ return errantRecordFuture;
+ }
+
+ public void waitForAllFutures() {
+ for (ErrantRecordFuture future : errantRecordFutures) {
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new ConnectException(e);
+ }
+ }
+ }
+
+ // Visible for testing
+ public class ErrantRecordFuture implements Future<Void> {
+
+ Future<RecordMetadata> future;
+
+ public ErrantRecordFuture(Future<RecordMetadata> producerFuture) {
+ future = producerFuture;
+ }
+
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ throw new UnsupportedOperationException("Reporting an errant record cannot be cancelled.");
+ }
+
+ public boolean isCancelled() {
+ return false;
+ }
+
+ public boolean isDone() {
+ return future == null || future.isDone();
+ }
+
+ public Void get() throws InterruptedException, ExecutionException {
+ if (future != null) {
Review comment:
any reason for this to ever be `null`?
##########
File path: connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporterTest.java
##########
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.connect.sink.SinkRecord;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("javax.management.*")
+public class WorkerErrantRecordReporterTest {
+
+ private WorkerErrantRecordReporter reporter;
+
+ private KafkaProducer<byte[], byte[]> producer = mock(KafkaProducer.class);
+ private SinkConnectorConfig sinkConnectorConfig = mock(SinkConnectorConfig.class);
+ private Converter converter = mock(Converter.class);
+ private HeaderConverter headerConverter = mock(HeaderConverter.class);
+ private SinkRecord record = mock(SinkRecord.class);
+
+ @Before
+ public void setup() {
+ reporter = new WorkerErrantRecordReporter(
+ producer,
+ sinkConnectorConfig,
+ converter,
+ converter,
+ headerConverter
+ );
+ }
+
+ @Test
+ public void testReport() {
+ when(sinkConnectorConfig.dlqTopicName()).thenReturn("dlq-topic");
+ when(sinkConnectorConfig.enableErrorLog()).thenReturn(false);
+ reporter.report(record, new Throwable());
Review comment:
maybe make this into a mock as well and at least assert on some method calls using `verify()`
##########
File path: connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporter.java
##########
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.header.Header;
+import org.apache.kafka.connect.header.Headers;
+import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter;
+import org.apache.kafka.connect.sink.ErrantRecordReporter;
+import org.apache.kafka.connect.sink.SinkRecord;
+
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class WorkerErrantRecordReporter implements ErrantRecordReporter {
+
+ private static final int DLQ_NUM_DESIRED_PARTITIONS = 1;
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerErrantRecordReporter.class);
+
+ private KafkaProducer<byte[], byte[]> producer;
+ private String dlqTopic;
+ private boolean useDlq;
+ private Converter keyConverter;
+ private Converter valueConverter;
+ private List<ErrantRecordFuture> errantRecordFutures;
+ private SinkConnectorConfig sinkConfig;
+ private HeaderConverter headerConverter;
+
+
Review comment:
2x
##########
File path: connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporter.java
##########
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.header.Header;
+import org.apache.kafka.connect.header.Headers;
+import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter;
+import org.apache.kafka.connect.sink.ErrantRecordReporter;
+import org.apache.kafka.connect.sink.SinkRecord;
+
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class WorkerErrantRecordReporter implements ErrantRecordReporter {
+
+ private static final int DLQ_NUM_DESIRED_PARTITIONS = 1;
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerErrantRecordReporter.class);
+
+ private KafkaProducer<byte[], byte[]> producer;
+ private String dlqTopic;
+ private boolean useDlq;
+ private Converter keyConverter;
+ private Converter valueConverter;
+ private List<ErrantRecordFuture> errantRecordFutures;
+ private SinkConnectorConfig sinkConfig;
+ private HeaderConverter headerConverter;
+
+
+ public static WorkerErrantRecordReporter createAndSetup(
+ Map<String, Object> adminProps,
+ Map<String, Object> producerProps,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+
+ KafkaProducer<byte[], byte[]> kafkaProducer = DeadLetterQueueReporter.setUpTopicAndProducer(
+ adminProps,
+ producerProps,
+ sinkConnectorConfig,
+ DLQ_NUM_DESIRED_PARTITIONS
+ );
+
+ return new WorkerErrantRecordReporter(
+ kafkaProducer,
+ sinkConnectorConfig,
+ workerKeyConverter,
+ workerValueConverter,
+ workerHeaderConverter
+ );
+ }
+
+ // Visible for testing purposes
+ public WorkerErrantRecordReporter(
+ KafkaProducer<byte[], byte[]> kafkaProducer,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+ producer = kafkaProducer;
+ dlqTopic = sinkConnectorConfig.dlqTopicName();
+ useDlq = dlqTopic != null && !dlqTopic.isEmpty();
+ keyConverter = workerKeyConverter;
+ valueConverter = workerValueConverter;
+ errantRecordFutures = new ArrayList<>();
+ sinkConfig = sinkConnectorConfig;
+ headerConverter = workerHeaderConverter;
+ }
+
+ @Override
+ public Future<Void> report(SinkRecord record, Throwable error) {
+
+ if (sinkConfig.enableErrorLog()) {
+ if (sinkConfig.includeRecordDetailsInErrorLog()) {
+ log.error("Error processing record: " + record.toString(), error);
Review comment:
```suggestion
log.error("Error processing record: {}", record.toString(), error);
```
##########
File path: connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java
##########
@@ -695,6 +705,32 @@ ErrorHandlingMetrics errorHandlingMetrics(ConnectorTaskId id) {
return reporters;
}
+ private WorkerErrantRecordReporter createWorkerErrantRecordReporter(
+ ConnectorTaskId id,
+ SinkConnectorConfig connConfig,
+ Class<? extends Connector> connectorClass,
+ Converter keyConverter,
+ Converter valueConverter,
+ HeaderConverter headerConverter
+ ) {
+ // check if errant record reporter topic is configured
+ String topic = connConfig.dlqTopicName();
+ if ((topic != null && !topic.isEmpty()) || connConfig.enableErrorLog()) {
Review comment:
would be nice if this was a method in the config
##########
File path: connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporter.java
##########
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.connect.errors.ConnectException;
+import org.apache.kafka.connect.header.Header;
+import org.apache.kafka.connect.header.Headers;
+import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter;
+import org.apache.kafka.connect.sink.ErrantRecordReporter;
+import org.apache.kafka.connect.sink.SinkRecord;
+
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class WorkerErrantRecordReporter implements ErrantRecordReporter {
+
+ private static final int DLQ_NUM_DESIRED_PARTITIONS = 1;
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerErrantRecordReporter.class);
+
+ private KafkaProducer<byte[], byte[]> producer;
+ private String dlqTopic;
+ private boolean useDlq;
+ private Converter keyConverter;
+ private Converter valueConverter;
+ private List<ErrantRecordFuture> errantRecordFutures;
+ private SinkConnectorConfig sinkConfig;
+ private HeaderConverter headerConverter;
+
+
+ public static WorkerErrantRecordReporter createAndSetup(
+ Map<String, Object> adminProps,
+ Map<String, Object> producerProps,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+
+ KafkaProducer<byte[], byte[]> kafkaProducer = DeadLetterQueueReporter.setUpTopicAndProducer(
+ adminProps,
+ producerProps,
+ sinkConnectorConfig,
+ DLQ_NUM_DESIRED_PARTITIONS
+ );
+
+ return new WorkerErrantRecordReporter(
+ kafkaProducer,
+ sinkConnectorConfig,
+ workerKeyConverter,
+ workerValueConverter,
+ workerHeaderConverter
+ );
+ }
+
+ // Visible for testing purposes
+ public WorkerErrantRecordReporter(
+ KafkaProducer<byte[], byte[]> kafkaProducer,
+ SinkConnectorConfig sinkConnectorConfig,
+ Converter workerKeyConverter,
+ Converter workerValueConverter,
+ HeaderConverter workerHeaderConverter
+ ) {
+ producer = kafkaProducer;
+ dlqTopic = sinkConnectorConfig.dlqTopicName();
+ useDlq = dlqTopic != null && !dlqTopic.isEmpty();
+ keyConverter = workerKeyConverter;
+ valueConverter = workerValueConverter;
+ errantRecordFutures = new ArrayList<>();
+ sinkConfig = sinkConnectorConfig;
+ headerConverter = workerHeaderConverter;
+ }
+
+ @Override
+ public Future<Void> report(SinkRecord record, Throwable error) {
+
+ if (sinkConfig.enableErrorLog()) {
+ if (sinkConfig.includeRecordDetailsInErrorLog()) {
+ log.error("Error processing record: " + record.toString(), error);
+ } else {
+ log.error(
+ "Error processing record in topic "
+ + record.topic()
+ + "at offset "
+ + record.kafkaOffset(),
+ error
+ );
+ }
+ }
+
+ Future<RecordMetadata> producerFuture = null;
+
+ if (useDlq) {
+
+ Headers headers = record.headers();
+ RecordHeaders result = new RecordHeaders();
+ if (headers != null) {
+ String topic = record.topic();
+ for (Header header : headers) {
+ String key = header.key();
+ byte[] rawHeader = headerConverter.fromConnectHeader(topic, key, header.schema(), header.value());
+ result.add(key, rawHeader);
+ }
+ }
+
+ ProducerRecord<byte[], byte[]> errantRecord = new ProducerRecord<>(
+ dlqTopic,
+ null,
+ record.timestamp() == RecordBatch.NO_TIMESTAMP ? record.timestamp() : null,
+ keyConverter.fromConnectData(dlqTopic, record.keySchema(), record.key()),
+ valueConverter.fromConnectData(dlqTopic, record.valueSchema(), record.value()),
+ result
+ );
+
+ producerFuture = producer.send(errantRecord);
+ }
+
+ ErrantRecordFuture errantRecordFuture = new ErrantRecordFuture(producerFuture);
+ errantRecordFutures.add(errantRecordFuture);
+ return errantRecordFuture;
+ }
+
+ public void waitForAllFutures() {
+ for (ErrantRecordFuture future : errantRecordFutures) {
+ try {
+ future.get();
Review comment:
should you remove the future from the list when you successfully `get()`?
##########
File path: connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
##########
@@ -214,6 +218,72 @@ public void testSourceConnector() throws Exception {
connect.deleteConnector(CONNECTOR_NAME);
}
+ @Test
+ public void testErrantRecordReporter() throws Exception {
+ connect.kafka().createTopic(DLQ_TOPIC, 1);
+ // create test topic
+ connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS);
+
+ // setup up props for the sink connector
+ Map<String, String> props = new HashMap<>();
+ props.put(CONNECTOR_CLASS_CONFIG, ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME);
+ props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
+ props.put(TOPICS_CONFIG, "test-topic");
Review comment:
extract to variable
##########
File path: connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
##########
@@ -214,6 +218,72 @@ public void testSourceConnector() throws Exception {
connect.deleteConnector(CONNECTOR_NAME);
}
+ @Test
+ public void testErrantRecordReporter() throws Exception {
+ connect.kafka().createTopic(DLQ_TOPIC, 1);
+ // create test topic
+ connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS);
+
+ // setup up props for the sink connector
+ Map<String, String> props = new HashMap<>();
+ props.put(CONNECTOR_CLASS_CONFIG, ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME);
+ props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
+ props.put(TOPICS_CONFIG, "test-topic");
+ props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+ props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+ props.put(DLQ_TOPIC_NAME_CONFIG, DLQ_TOPIC);
+
+ // expect all records to be consumed by the connector
+ connectorHandle.expectedRecords(NUM_RECORDS_PRODUCED);
+
+ // expect all records to be consumed by the connector
+ connectorHandle.expectedCommits(NUM_RECORDS_PRODUCED);
+
+ // validate the intended connector configuration, a config that errors
+ connect.assertions().assertExactlyNumErrorsOnConnectorConfigValidation(ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME, props, 1,
+ "Validating connector configuration produced an unexpected number or errors.");
+
+ // add missing configuration to make the config valid
+ props.put("name", CONNECTOR_NAME);
+
+ // validate the intended connector configuration, a valid config
+ connect.assertions().assertExactlyNumErrorsOnConnectorConfigValidation(ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME, props, 0,
+ "Validating connector configuration produced an unexpected number or errors.");
+
+ // start a sink connector
+ connect.configureConnector(CONNECTOR_NAME, props);
+
+ waitForCondition(this::checkForPartitionAssignment,
+ CONNECTOR_SETUP_DURATION_MS,
+ "Connector tasks were not assigned a partition each.");
Review comment:
```suggestion
waitForCondition(
this::checkForPartitionAssignment,
CONNECTOR_SETUP_DURATION_MS,
"Connector tasks were not assigned a partition each."
);
```
##########
File path: connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExampleConnectIntegrationTest.java
##########
@@ -214,6 +218,72 @@ public void testSourceConnector() throws Exception {
connect.deleteConnector(CONNECTOR_NAME);
}
+ @Test
+ public void testErrantRecordReporter() throws Exception {
+ connect.kafka().createTopic(DLQ_TOPIC, 1);
+ // create test topic
+ connect.kafka().createTopic("test-topic", NUM_TOPIC_PARTITIONS);
+
+ // setup up props for the sink connector
+ Map<String, String> props = new HashMap<>();
+ props.put(CONNECTOR_CLASS_CONFIG, ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME);
+ props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS));
+ props.put(TOPICS_CONFIG, "test-topic");
+ props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+ props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName());
+ props.put(DLQ_TOPIC_NAME_CONFIG, DLQ_TOPIC);
+
+ // expect all records to be consumed by the connector
+ connectorHandle.expectedRecords(NUM_RECORDS_PRODUCED);
+
+ // expect all records to be consumed by the connector
+ connectorHandle.expectedCommits(NUM_RECORDS_PRODUCED);
+
+ // validate the intended connector configuration, a config that errors
+ connect.assertions().assertExactlyNumErrorsOnConnectorConfigValidation(ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME, props, 1,
+ "Validating connector configuration produced an unexpected number or errors.");
+
+ // add missing configuration to make the config valid
+ props.put("name", CONNECTOR_NAME);
+
+ // validate the intended connector configuration, a valid config
+ connect.assertions().assertExactlyNumErrorsOnConnectorConfigValidation(ERRANT_RECORD_SINK_CONNECTOR_CLASS_NAME, props, 0,
+ "Validating connector configuration produced an unexpected number or errors.");
Review comment:
do we need this invalid config step here
##########
File path: connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerErrantRecordReporterTest.java
##########
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kafka.connect.runtime;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.connect.sink.SinkRecord;
+import org.apache.kafka.connect.storage.Converter;
+import org.apache.kafka.connect.storage.HeaderConverter;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("javax.management.*")
+public class WorkerErrantRecordReporterTest {
+
+ private WorkerErrantRecordReporter reporter;
+
Review comment:
```suggestion
```
nit
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org