You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2024/06/24 23:06:09 UTC
(nifi) 01/03: NIFI-11259 Added Redesigned Kafka Processors and Services
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 24925ed9b58da0cdab309d93ef73d953b1f9fa3d
Author: Paul Grey <gr...@yahoo.com>
AuthorDate: Tue Mar 28 17:48:34 2023 -0400
NIFI-11259 Added Redesigned Kafka Processors and Services
- Added ConsumeKafka and PublishKafka Processors with Kafka3ConnectionService
- Added Kafka 3 integration tests
This closes #8463
Co-authored-by: David Handermann <ex...@apache.org>
Signed-off-by: David Handermann <ex...@apache.org>
---
nifi-assembly/pom.xml | 18 +
.../nifi-kafka-2-6-integration/pom.xml | 89 ++++
.../consume/it/ConsumeKafkaDemarcator_2_6_IT.java | 146 ++++++
.../consume/it/ConsumeKafkaRecord_2_6_IT.java | 105 ++++
.../it/ConsumeKafkaWrapperRecord_2_6_IT.java | 172 +++++++
.../kafka/consume/it/ConsumeKafka_2_6_BaseIT.java | 140 ++++++
.../kafka/consume/it/ConsumeKafka_2_6_IT.java | 134 ++++++
.../publish/it/PublishKafkaRecord_2_6_IT.java | 101 ++++
.../kafka/publish/it/PublishKafka_2_6_BaseIT.java | 84 ++++
.../kafka/publish/it/PublishKafka_2_6_IT.java | 80 +++
.../it/PublishKafka_2_6_PartitionStrategyIT.java | 68 +++
.../processors/kafka/pubsub/KafkaProcessor.java} | 35 +-
.../apache/nifi/processors/kafka/publish/ff.json | 1 +
.../src/test/resources/simplelogger.properties | 20 +
.../processors/kafka/pubsub/ConsumerLease.java | 2 +-
.../nifi-kafka-3-integration/README.md | 12 +
.../nifi-kafka-3-integration/pom.xml | 125 +++++
.../kafka/processors/AbstractConsumeKafkaIT.java | 76 +++
.../nifi/kafka/processors/AbstractKafkaBaseIT.java | 124 +++++
.../kafka/processors/AbstractPublishKafkaIT.java} | 28 +-
.../kafka/processors/ConsumeKafkaDemarcatorIT.java | 229 +++++++++
.../nifi/kafka/processors/ConsumeKafkaIT.java | 167 +++++++
.../processors/ConsumeKafkaKeyAttributeIT.java | 97 ++++
.../kafka/processors/ConsumeKafkaOffsetsIT.java | 110 +++++
.../kafka/processors/ConsumeKafkaRecordIT.java | 146 ++++++
.../processors/ConsumeKafkaServiceLifecycleIT.java | 52 ++
.../processors/ConsumeKafkaWrapperRecordIT.java | 180 +++++++
.../processors/PublishKafkaAcksBestEffortIT.java | 87 ++++
.../processors/PublishKafkaAcksOneNodeIT.java | 65 +++
.../PublishKafkaAuthSaslPlaintextIT.java | 157 ++++++
.../processors/PublishKafkaCompressGZIPIT.java | 67 +++
.../kafka/processors/PublishKafkaDelimitedIT.java | 81 ++++
.../processors/PublishKafkaFailureStrategyIT.java | 76 +++
.../nifi/kafka/processors/PublishKafkaIT.java | 78 +++
.../PublishKafkaPartitionStrategyIT.java | 64 +++
.../kafka/processors/PublishKafkaRecordIT.java | 93 ++++
.../PublishKafkaRecordPartialHandlingIT.java | 101 ++++
.../nifi/kafka/processors/PublishKafkaSSLIT.java | 90 ++++
.../processors/PublishKafkaValueRecordIT.java | 109 +++++
.../processors/PublishKafkaWrapperRecordIT.java | 109 +++++
.../publish/PublishKafkaMultipleFFIT.java | 128 +++++
.../additional/PublishKafkaContentX1IT.java | 104 ++++
.../additional/PublishKafkaWrapperX1IT.java | 95 ++++
.../additional/PublishKafkaWrapperX2IT.java | 101 ++++
.../additional/PublishKafkaWrapperX3IT.java | 98 ++++
.../additional/PublishKafkaWrapperX4IT.java | 101 ++++
.../additional/PublishKafkaWrapperX5IT.java | 142 ++++++
.../kafka/processors/publish/additional/README.md | 18 +
.../dependency/PublishKafkaMultipleFFIT.java | 165 +++++++
.../processors/publish/additional/contentX1.json | 1 +
.../processors/publish/additional/wrapperX1.json | 15 +
.../processors/publish/additional/wrapperX2.json | 15 +
.../processors/publish/additional/wrapperX3.json | 14 +
.../processors/publish/additional/wrapperX4.json | 16 +
.../processors/publish/additional/wrapperX5.json | 17 +
.../apache/nifi/kafka/processors/publish/ff.json | 1 +
.../nifi/kafka/processors/publish/ff.not.json | 1 +
.../nifi/kafka/processors/publish/ff.partial.json | 8 +
.../nifi/kafka/processors/publish/ffvalue.json | 26 +
.../nifi/kafka/processors/publish/ffwrapper.json | 53 ++
.../src/test/resources/simplelogger.properties | 22 +
.../nifi-kafka-3-service-nar/pom.xml | 41 ++
.../nifi-kafka-bundle/nifi-kafka-3-service/pom.xml | 96 ++++
.../kafka/service/Kafka3ConnectionService.java | 387 +++++++++++++++
.../kafka/service/consumer/ConsumerException.java} | 29 +-
.../service/consumer/Kafka3ConsumerService.java | 205 ++++++++
.../service/consumer/pool/ConsumerFactory.java} | 38 +-
.../service/consumer/pool/ConsumerObjectPool.java} | 40 +-
.../consumer/pool/ConsumerPooledObjectFactory.java | 98 ++++
.../consumer/pool/StandardConsumerFactory.java | 42 ++
.../kafka/service/consumer/pool/Subscription.java | 120 +++++
.../service/producer/Kafka3ProducerService.java | 121 +++++
.../kafka/service/producer/ProducerCallback.java | 87 ++++
.../KafkaNonTransactionalProducerWrapper.java} | 36 +-
.../producer/transaction/KafkaProducerWrapper.java | 81 ++++
.../KafkaTransactionalProducerWrapper.java} | 57 +--
.../org.apache.nifi.controller.ControllerService | 15 +
.../service/Kafka3ConnectionServiceBaseIT.java | 346 +++++++++++++
.../service/Kafka3ConnectionServiceSSLIT.java | 66 +++
.../Kafka3ConnectionServiceSaslPlaintextIT.java | 57 +++
.../nifi/kafka/service/KafkaConnectivityIT.java | 100 ++++
.../pool/ConsumerPooledObjectFactoryTest.java | 88 ++++
.../src/test/resources/simplelogger.properties | 20 +
.../nifi-kafka-code-coverage/README.md | 20 +
.../nifi-kafka-code-coverage/pom.xml | 102 ++++
.../nifi-kafka-bundle/nifi-kafka-nar/pom.xml | 41 ++
.../nifi-kafka-processors/pom.xml | 65 +++
.../apache/nifi/kafka/processors/ConsumeKafka.java | 477 ++++++++++++++++++
.../apache/nifi/kafka/processors/PublishKafka.java | 536 +++++++++++++++++++++
.../nifi/kafka/processors/common/KafkaUtils.java | 107 ++++
.../kafka/processors/consumer/OffsetTracker.java | 53 ++
.../processors/consumer/ProcessingStrategy.java | 51 ++
.../processors/consumer/bundle/BundleKey.java | 93 ++++
.../processors/consumer/bundle/BundleValue.java | 58 +++
.../consumer/bundle/ByteRecordBundler.java | 97 ++++
.../FlowFileStreamKafkaMessageConverter.java | 78 +++
.../consumer/convert/KafkaMessageConverter.java} | 33 +-
.../convert/RecordStreamKafkaMessageConverter.java | 128 +++++
.../WrapperRecordStreamKafkaMessageConverter.java | 152 ++++++
.../consumer/wrapper/ConsumeWrapperRecord.java | 91 ++++
.../consumer/wrapper/WrapperRecordKeyReader.java | 104 ++++
.../processors/producer/PartitionStrategy.java | 63 +++
.../processors/producer/common/ProducerUtils.java} | 32 +-
.../producer/common/PublishKafkaUtil.java | 79 +++
.../producer/config/DeliveryGuarantee.java | 58 +++
.../DelimitedStreamKafkaRecordConverter.java | 59 +++
.../FlowFileStreamKafkaRecordConverter.java | 54 +++
.../producer/convert/KafkaRecordConverter.java} | 34 +-
.../convert/RecordStreamKafkaRecordConverter.java | 124 +++++
.../RecordWrapperStreamKafkaRecordConverter.java | 140 ++++++
.../producer/header/AttributesHeadersFactory.java | 50 ++
.../producer/header/HeadersFactory.java} | 33 +-
.../producer/key/AttributeKeyFactory.java | 41 ++
.../kafka/processors/producer/key/KeyFactory.java} | 33 +-
.../processors/producer/key/MessageKeyFactory.java | 50 ++
.../producer/value/RecordValueFactory.java} | 52 +-
.../processors/producer/value/ValueFactory.java} | 32 +-
.../producer/wrapper/RecordFieldConverter.java | 103 ++++
.../producer/wrapper/RecordMetadataStrategy.java | 58 +++
.../processors/producer/wrapper/WrapperRecord.java | 100 ++++
.../services/org.apache.nifi.processor.Processor | 16 +
.../nifi/kafka/processors/ConsumeKafkaTest.java | 126 +++++
.../nifi/kafka/processors/PublishKafkaTest.java | 121 +++++
.../producer/key/AttributeKeyFactoryTest.java | 74 +++
.../queueIn/PublishKafkaNoFlowFileTest.java | 68 +++
.../producer/wrapper/WrapperRecordTest.java | 101 ++++
.../nifi-kafka-service-api-nar/pom.xml | 41 ++
.../{ => nifi-kafka-service-api}/pom.xml | 30 +-
.../kafka/service/api/KafkaConnectionService.java} | 32 +-
.../kafka/service/api/common/OffsetSummary.java} | 34 +-
.../kafka/service/api/common/PartitionState.java} | 37 +-
.../service/api/common/ServiceConfiguration.java} | 33 +-
.../service/api/common/TopicPartitionSummary.java | 54 +++
.../service/api/consumer/AutoOffsetReset.java | 54 +++
.../api/consumer/ConsumerConfiguration.java} | 28 +-
.../service/api/consumer/KafkaConsumerService.java | 50 ++
.../kafka/service/api/consumer/PollingContext.java | 83 ++++
.../kafka/service/api/consumer/PollingSummary.java | 56 +++
.../kafka/service/api/header/RecordHeader.java | 56 +++
.../kafka/service/api/producer/FlowFileResult.java | 55 +++
.../service/api/producer/KafkaProducerService.java | 58 +++
.../api/producer/ProducerConfiguration.java | 58 +++
.../api/producer/ProducerRecordMetadata.java} | 57 +--
.../kafka/service/api/producer/PublishContext.java | 64 +++
.../kafka/service/api/producer/RecordSummary.java} | 36 +-
.../nifi/kafka/service/api/record/ByteRecord.java | 88 ++++
.../nifi/kafka/service/api/record/KafkaRecord.java | 74 +++
.../service/api/producer/PublishContextTest.java | 43 ++
.../shared/attribute/KafkaFlowFileAttribute.java | 2 +
.../nifi/kafka/shared/property/IsolationLevel.java | 52 ++
.../apache/nifi/kafka/shared/util/Notifier.java | 67 +++
nifi-extension-bundles/nifi-kafka-bundle/pom.xml | 31 +-
152 files changed, 11763 insertions(+), 526 deletions(-)
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
index 87747410f3..582cf56714 100644
--- a/nifi-assembly/pom.xml
+++ b/nifi-assembly/pom.xml
@@ -298,6 +298,24 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api-nar</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-service-nar</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-nar</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-confluent-platform-nar</artifactId>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/pom.xml
new file mode 100644
index 0000000000..debafc092c
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-2-6-integration</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-schema-registry-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kerberos-user-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-services</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>kafka</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-2-6-processors</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaDemarcator_2_6_IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaDemarcator_2_6_IT.java
new file mode 100644
index 0000000000..683750c3d5
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaDemarcator_2_6_IT.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.consume.it;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ConsumeKafkaDemarcator_2_6_IT extends ConsumeKafka_2_6_BaseIT {
+ private static final String TEST_RECORD_VALUE = "recordA,recordB,recordC,recordD,recordE,recordF";
+ private static final int EXPECTED_TOKENS = 6;
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setUp() {
+ runner = TestRunners.newTestRunner(ConsumeKafka_2_6.class);
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ runner.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ }
+
+ @Test
+ public void testConsumeDemarcated() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, 4);
+ runner.setProperty("group.id", groupId);
+ runner.setProperty("topic", topic);
+ runner.setProperty("message-demarcator", ",");
+ runner.setProperty("separate-by-key", Boolean.FALSE.toString());
+ runner.run(1, false, true);
+
+ final Collection<ProducerRecord<String, String>> records = new ArrayList<>();
+ final String[] values = TEST_RECORD_VALUE.split(",");
+ for (String value : values) {
+ records.add(new ProducerRecord<>(topic, value));
+ }
+ produce(topic, records);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship("success");
+ assertEquals(1, flowFiles.size());
+ final MockFlowFile flowFile = flowFiles.getFirst();
+ assertEquals(TEST_RECORD_VALUE, flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(EXPECTED_TOKENS - 1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(EXPECTED_TOKENS));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+ }
+
+ @Test
+ public void testConsumeDemarcatedSeparateByKey() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, 4);
+ runner.setProperty("group.id", groupId);
+ runner.setProperty("topic", topic);
+ runner.setProperty("message-demarcator", ",");
+ runner.setProperty("separate-by-key", Boolean.TRUE.toString());
+ runner.run(1, false, true);
+
+ final Collection<ProducerRecord<String, String>> records = new ArrayList<>();
+ final String[] values = TEST_RECORD_VALUE.split(",");
+ boolean key = false;
+ for (String value : values) {
+ records.add(new ProducerRecord<>(topic, Boolean.toString(key), value));
+ key = !key;
+ }
+ produce(topic, records);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 2); // key=false, key=true
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship("success");
+ assertEquals(2, flowFiles.size());
+ for (MockFlowFile flowFile : flowFiles) {
+ if (Boolean.TRUE.toString().equals(flowFile.getAttribute(KafkaFlowFileAttribute.KAFKA_KEY))) {
+ assertEquals("recordB,recordD,recordF", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(EXPECTED_TOKENS - 1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(EXPECTED_TOKENS / 2));
+ } else if (Boolean.FALSE.toString().equals(flowFile.getAttribute(KafkaFlowFileAttribute.KAFKA_KEY))) {
+ assertEquals("recordA,recordC,recordE", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(EXPECTED_TOKENS - 2));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(EXPECTED_TOKENS / 2));
+ } else {
+ Assertions.fail("expected KafkaFlowFileAttribute.KAFKA_KEY");
+ }
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(0));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaRecord_2_6_IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaRecord_2_6_IT.java
new file mode 100644
index 0000000000..0a4f24c7e4
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaRecord_2_6_IT.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.consume.it;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ConsumeKafkaRecord_2_6_IT extends ConsumeKafka_2_6_BaseIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/processors/kafka/publish/ff.json";
+ private static final int TEST_RECORD_COUNT = 3;
+
+ private static final String TOPIC = ConsumeKafkaRecord_2_6_IT.class.getName();
+ private static final String GROUP_ID = ConsumeKafkaRecord_2_6_IT.class.getSimpleName();
+
+ @Test
+ public void testKafkaTestContainerProduceConsumeOne() throws ExecutionException, InterruptedException, IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(ConsumeKafkaRecord_2_6.class);
+ runner.setValidateExpressionUsage(false);
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ runner.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ runner.setProperty("topic", TOPIC);
+ runner.setProperty("group.id", GROUP_ID);
+ runner.run(1, false, true);
+
+ final String message = new String(IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE))), StandardCharsets.UTF_8);
+ produceOne(TOPIC, null, null, message, Collections.emptyList());
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship("success");
+ assertEquals(1, flowFiles.size());
+ final MockFlowFile flowFile = flowFiles.getFirst();
+
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, TOPIC);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(0));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+ flowFile.assertAttributeEquals("record.count", Long.toString(3));
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+
+ // [{"id":1,"name":"A"},{"id":2,"name":"B"},{"id":3,"name":"C"}]
+ final JsonNode jsonNodeTree = objectMapper.readTree(flowFile.getContent());
+ assertInstanceOf(ArrayNode.class, jsonNodeTree);
+ final ArrayNode arrayNode = (ArrayNode) jsonNodeTree;
+ final Iterator<JsonNode> elements = arrayNode.elements();
+ assertEquals(TEST_RECORD_COUNT, arrayNode.size());
+ while (elements.hasNext()) {
+ final JsonNode jsonNode = elements.next();
+ assertTrue(Arrays.asList(1, 2, 3).contains(jsonNode.get("id").asInt()));
+ assertTrue(Arrays.asList("A", "B", "C").contains(jsonNode.get("name").asText()));
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaWrapperRecord_2_6_IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaWrapperRecord_2_6_IT.java
new file mode 100644
index 0000000000..9754ff4cac
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafkaWrapperRecord_2_6_IT.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.consume.it;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.shared.property.KeyFormat;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6;
+import org.apache.nifi.processors.kafka.pubsub.KafkaProcessor;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ConsumeKafkaWrapperRecord_2_6_IT extends ConsumeKafka_2_6_BaseIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/processors/kafka/publish/ff.json";
+ private static final int TEST_RECORD_COUNT = 3;
+ private static final String MESSAGE_KEY = "{\"id\": 0,\"name\": \"K\"}";
+
+ static abstract class Verifier {
+ abstract void verify(final JsonNode jsonNode);
+ }
+
+ static class RecordVerifier extends Verifier {
+ void verify(final JsonNode jsonNode) {
+ final ObjectNode key = assertInstanceOf(ObjectNode.class, jsonNode);
+ assertEquals(2, key.size());
+ assertEquals(0, key.get("id").asInt());
+ assertEquals("K", key.get("name").asText());
+ }
+ }
+
+ static class StringVerifier extends Verifier {
+ void verify(final JsonNode jsonNode) {
+ final TextNode key = assertInstanceOf(TextNode.class, jsonNode);
+ assertEquals(MESSAGE_KEY, key.asText());
+ }
+ }
+
+ static class ByteArrayVerifier extends Verifier {
+ void verify(final JsonNode jsonNode) {
+ final ArrayNode key = assertInstanceOf(ArrayNode.class, jsonNode);
+ assertEquals(MESSAGE_KEY.length(), key.size());
+ for (int i = 0; (i < MESSAGE_KEY.length()); ++i) {
+ assertEquals(MESSAGE_KEY.charAt(i), key.get(i).asInt());
+ }
+ }
+ }
+
+ public static Stream<Arguments> permutationsKeyFormat() {
+ return Stream.of(
+ Arguments.arguments(KeyFormat.RECORD, new RecordVerifier()),
+ Arguments.arguments(KeyFormat.STRING, new StringVerifier()),
+ Arguments.arguments(KeyFormat.BYTE_ARRAY, new ByteArrayVerifier())
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("permutationsKeyFormat")
+ void testWrapperRecord(final KeyFormat keyFormat, final Verifier verifier)
+ throws InterruptedException, ExecutionException, IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(ConsumeKafkaRecord_2_6.class);
+ runner.setValidateExpressionUsage(false);
+
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ addKeyRecordReaderService(runner);
+ runner.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ runner.setProperty(KafkaProcessor.OUTPUT_STRATEGY, OutputStrategy.USE_WRAPPER);
+ runner.setProperty(KafkaProcessor.KEY_FORMAT, keyFormat);
+ runner.setProperty("topic", topic);
+ runner.setProperty("group.id", groupId);
+
+ runner.run(1, false, true);
+ final String message = new String(IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE))), StandardCharsets.UTF_8);
+ final List<Header> headersPublish = Collections.singletonList(
+ new RecordHeader("header1", "value1".getBytes(StandardCharsets.UTF_8)));
+ produceOne(topic, 0, MESSAGE_KEY, message, headersPublish);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+ runner.run(1, true, false);
+
+ runner.assertAllFlowFilesTransferred(KafkaProcessor.REL_SUCCESS, 1);
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(KafkaProcessor.REL_SUCCESS);
+ assertEquals(1, flowFiles.size());
+ final MockFlowFile flowFile = flowFiles.getFirst();
+ runner.getLogger().trace(flowFile.getContent());
+ flowFile.assertAttributeEquals("record.count", Long.toString(TEST_RECORD_COUNT));
+
+ final ObjectMapper objectMapper = new ObjectMapper();
+ final JsonNode jsonNodeTree = objectMapper.readTree(flowFile.getContent());
+ final ArrayNode arrayNode = assertInstanceOf(ArrayNode.class, jsonNodeTree);
+
+ final Iterator<JsonNode> elements = arrayNode.elements();
+ assertEquals(TEST_RECORD_COUNT, arrayNode.size());
+ while (elements.hasNext()) {
+ final ObjectNode wrapper = assertInstanceOf(ObjectNode.class, elements.next());
+
+ verifier.verify(wrapper.get("key"));
+
+ final ObjectNode value = assertInstanceOf(ObjectNode.class, wrapper.get("value"));
+ assertTrue(Arrays.asList(1, 2, 3).contains(value.get("id").asInt()));
+ assertTrue(Arrays.asList("A", "B", "C").contains(value.get("name").asText()));
+
+ final ObjectNode headers = assertInstanceOf(ObjectNode.class, wrapper.get("headers"));
+ assertEquals(1, headers.size());
+ assertEquals("value1", headers.get("header1").asText());
+
+ final ObjectNode metadata = assertInstanceOf(ObjectNode.class, wrapper.get("metadata"));
+ assertEquals(topic, metadata.get("topic").asText());
+ assertEquals(0, metadata.get("partition").asInt());
+ assertEquals(0, metadata.get("offset").asInt());
+ assertTrue(metadata.get("timestamp").isIntegralNumber());
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafka_2_6_BaseIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafka_2_6_BaseIT.java
new file mode 100644
index 0000000000..d649ec4b32
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafka_2_6_BaseIT.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.consume.it;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.nifi.json.JsonRecordSetWriter;
+import org.apache.nifi.json.JsonTreeReader;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.util.TestRunner;
+import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ConsumeKafka_2_6_BaseIT {
+ // https://docs.confluent.io/platform/current/installation/versions-interoperability.html#cp-and-apache-kafka-compatibility
+ protected static final String IMAGE_NAME = "confluentinc/cp-kafka:6.1.14"; // Kafka 2.7
+
+ protected static final Duration DURATION_POLL = Duration.ofMillis(1000L);
+
+ protected static final KafkaContainer kafka;
+
+ protected static ObjectMapper objectMapper;
+
+ // https://www.testcontainers.org/test_framework_integration/manual_lifecycle_control/
+ static {
+ /*
+ * NIFI-11259
+ * Consider abstracting this configuration to a base class shared with
+ * {@link org.apache.nifi.processors.kafka.publish.it.PublishKafka_2_6_BaseIT}, such that only one testcontainer
+ * is needed to service all module ITs.
+ */
+ kafka = new KafkaContainer(DockerImageName.parse(IMAGE_NAME));
+ kafka.start();
+ }
+
+ @BeforeAll
+ protected static void beforeAll() {
+ objectMapper = new ObjectMapper();
+ }
+
+ protected void addRecordReaderService(final TestRunner runner) throws InitializationException {
+ final String readerId = "record-reader";
+ final RecordReaderFactory readerService = new JsonTreeReader();
+ runner.addControllerService(readerId, readerService);
+ runner.enableControllerService(readerService);
+ runner.setProperty(readerId, readerId);
+ }
+
+ protected void addRecordWriterService(final TestRunner runner) throws InitializationException {
+ final String writerId = "record-writer";
+ final RecordSetWriterFactory writerService = new JsonRecordSetWriter();
+ runner.addControllerService(writerId, writerService);
+ runner.enableControllerService(writerService);
+ runner.setProperty(writerId, writerId);
+ }
+
+ protected void addKeyRecordReaderService(final TestRunner runner) throws InitializationException {
+ final String readerId = "key-record-reader";
+ final RecordReaderFactory readerService = new JsonTreeReader();
+ runner.addControllerService(readerId, readerService);
+ runner.enableControllerService(readerService);
+ runner.setProperty(readerId, readerId);
+ }
+
+ protected Properties getProducerProperties() {
+ final Properties properties = new Properties();
+ properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
+ properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ // use of these two properties is intended to induce quick send of test events, so they are available to be consumed
+ // final long maxDelayBeforeSend = 100L;
+ // properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, Long.toString(maxDelayBeforeSend));
+ // properties.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, Long.toString(maxDelayBeforeSend));
+ return properties;
+ }
+
+ protected void produceOne(final String topic, final Integer partition,
+ final String key, final String value, final List<Header> headers)
+ throws ExecutionException, InterruptedException {
+ try (final KafkaProducer<String, String> producer = new KafkaProducer<>(getProducerProperties())) {
+ final ProducerRecord<String, String> record = new ProducerRecord<>(topic, partition, key, value, headers);
+ final Future<RecordMetadata> future = producer.send(record);
+ final RecordMetadata metadata = future.get();
+ assertEquals(topic, metadata.topic());
+ assertTrue(metadata.hasOffset());
+ assertEquals(0L, metadata.offset());
+ }
+ }
+
+ protected void produce(final String topic, final Collection<ProducerRecord<String, String>> records)
+ throws ExecutionException, InterruptedException {
+ final Collection<RecordMetadata> metadatas = new ArrayList<>();
+ try (final KafkaProducer<String, String> producer = new KafkaProducer<>(getProducerProperties())) {
+ final Collection<Future<RecordMetadata>> futures = records.stream().map(producer::send).collect(Collectors.toList());
+ for (final Future<RecordMetadata> future : futures) {
+ metadatas.add(future.get());
+ }
+ assertEquals(records.size(), futures.size());
+ assertEquals(futures.size(), metadatas.size());
+ }
+ for (RecordMetadata metadata : metadatas) {
+ assertEquals(topic, metadata.topic());
+ assertTrue(metadata.hasOffset());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafka_2_6_IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafka_2_6_IT.java
new file mode 100644
index 0000000000..734390060b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/consume/it/ConsumeKafka_2_6_IT.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.consume.it;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ConsumeKafka_2_6_IT extends ConsumeKafka_2_6_BaseIT {
+ private static final String TEST_RECORD_KEY = "key-" + System.currentTimeMillis();
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setUp() {
+ runner = TestRunners.newTestRunner(ConsumeKafka_2_6.class);
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ runner.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ }
+
+ @Test
+ public void testProduceConsumeOne() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, 4);
+ runner.setProperty("topic", topic);
+ runner.setProperty("group.id", groupId);
+ runner.run(1, false, true);
+
+ produceOne(topic, null, TEST_RECORD_KEY, TEST_RECORD_VALUE, Collections.emptyList());
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship("success");
+ assertEquals(1, flowFiles.size());
+ final MockFlowFile flowFile = flowFiles.getFirst();
+
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(0));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+ }
+
+ /**
+ * Test ability to specify a topic regular expression to query for messages.
+ */
+ @Test
+ public void testTopicPattern() throws ExecutionException, InterruptedException {
+ final String topicBase = UUID.randomUUID().toString();
+ final String topicTestCase = topicBase + "-B";
+ final String topicPattern = topicBase + ".*";
+
+ // on use of "pattern" subscription, things seem to work better when topic exists prior to the subscribe event
+ produceOne(topicTestCase, null, TEST_RECORD_KEY, TEST_RECORD_VALUE, Collections.emptyList());
+
+ runner.setProperty("topic", topicPattern);
+ runner.setProperty("topic_type", "pattern");
+ runner.setProperty("group.id", "B");
+ runner.run(1, false, true);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+ }
+
+ /**
+ * Test ability to specify multiple topics to query for messages.
+ */
+ @Test
+ public void testTopicNames() throws ExecutionException, InterruptedException {
+ final String topicBase = UUID.randomUUID().toString();
+ final String topicTestCase = topicBase + "-C";
+ final String topicNames = topicBase + "-D," + topicTestCase;
+
+ runner.setProperty("topic", topicNames);
+ runner.setProperty("topic_type", "names");
+ runner.setProperty("group.id", "C");
+ runner.run(1, false, true);
+
+ produceOne(topicTestCase, null, TEST_RECORD_KEY, TEST_RECORD_VALUE, Collections.emptyList());
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafkaRecord_2_6_IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafkaRecord_2_6_IT.java
new file mode 100644
index 0000000000..c8e9cc7cc0
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafkaRecord_2_6_IT.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.publish.it;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaRecord_2_6_IT extends PublishKafka_2_6_BaseIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/processors/kafka/publish/ff.json";
+
+ private static final int TEST_RECORD_COUNT = 3;
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafkaRecord_2_6.class);
+ runner.setValidateExpressionUsage(false);
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ runner.setProperty("bootstrap.servers", String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setProperty("topic", getClass().getName());
+
+ runner.setProperty("message-key-field", "name");
+ runner.setProperty("attribute-name-regex", "a.*");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test_2_ConsumeMultipleRecords() throws JsonProcessingException {
+ final Pattern pattern = Pattern.compile("[ABC]");
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(TEST_RECORD_COUNT, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ assertTrue(pattern.matcher(record.key()).matches(),
+ String.format("%s does not match %s", record.key(), pattern.pattern()));
+ final JsonNode jsonNode = objectMapper.readTree(record.value());
+ assertNotNull(jsonNode);
+ assertEquals(2, jsonNode.size());
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_BaseIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_BaseIT.java
new file mode 100644
index 0000000000..550515bf19
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_BaseIT.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.publish.it;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.nifi.json.JsonRecordSetWriter;
+import org.apache.nifi.json.JsonTreeReader;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.util.TestRunner;
+import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.Properties;
+
+public class PublishKafka_2_6_BaseIT {
+ // https://docs.confluent.io/platform/current/installation/versions-interoperability.html#cp-and-apache-kafka-compatibility
+ public static final String IMAGE_NAME = "confluentinc/cp-kafka:6.1.14"; // Kafka 2.7
+
+ public static final Duration DURATION_POLL = Duration.ofMillis(1000L);
+
+ protected static final KafkaContainer kafka;
+
+ protected static ObjectMapper objectMapper;
+
+ // https://www.testcontainers.org/test_framework_integration/manual_lifecycle_control/
+ static {
+ kafka = new KafkaContainer(DockerImageName.parse(IMAGE_NAME));
+ kafka.start();
+ }
+
+ @BeforeAll
+ protected static void beforeAll() {
+ objectMapper = new ObjectMapper();
+ }
+
+ protected void addRecordReaderService(final TestRunner runner) throws InitializationException {
+ final String readerId = "record-reader";
+ final RecordReaderFactory readerService = new JsonTreeReader();
+ runner.addControllerService(readerId, readerService);
+ runner.enableControllerService(readerService);
+ runner.setProperty(readerId, readerId);
+ }
+
+ protected void addRecordWriterService(final TestRunner runner) throws InitializationException {
+ final String writerId = "record-writer";
+ final RecordSetWriterFactory writerService = new JsonRecordSetWriter();
+ runner.addControllerService(writerId, writerService);
+ runner.enableControllerService(writerService);
+ runner.setProperty(writerId, writerId);
+ }
+
+ protected Properties getKafkaConsumerProperties() {
+ final Properties properties = new Properties();
+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
+ properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, Boolean.TRUE.toString());
+ properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
+ properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ return properties;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_IT.java
new file mode 100644
index 0000000000..32a7226096
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_IT.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.publish.it;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.processors.kafka.pubsub.PublishKafka_2_6;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafka_2_6_IT extends PublishKafka_2_6_BaseIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka_2_6.class);
+ runner.setValidateExpressionUsage(false);
+
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ runner.setProperty("bootstrap.servers", String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setProperty("topic", getClass().getName());
+ runner.setProperty("attribute-name-regex", "a.*");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+
+ runner.enqueue(TEST_RECORD_VALUE, attributes);
+ runner.run();
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_PartitionStrategyIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_PartitionStrategyIT.java
new file mode 100644
index 0000000000..e83ee4fc46
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/publish/it/PublishKafka_2_6_PartitionStrategyIT.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.kafka.publish.it;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.nifi.processors.kafka.pubsub.Partitioners;
+import org.apache.nifi.processors.kafka.pubsub.PublishKafka_2_6;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.net.URI;
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafka_2_6_PartitionStrategyIT extends PublishKafka_2_6_BaseIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka_2_6.class);
+ runner.setValidateExpressionUsage(false);
+
+ final URI uri = URI.create(kafka.getBootstrapServers());
+ runner.setProperty("bootstrap.servers", String.format("%s:%s", uri.getHost(), uri.getPort()));
+ runner.setProperty("topic", getClass().getName());
+ runner.setProperty(ProducerConfig.PARTITIONER_CLASS_CONFIG, Partitioners.ExpressionLanguagePartitioner.class.getName());
+ runner.setProperty("partition", Long.toString(System.currentTimeMillis()));
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.run();
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/pubsub/KafkaProcessor.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/pubsub/KafkaProcessor.java
index 15aa16ed76..47e9c998f0 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/java/org/apache/nifi/processors/kafka/pubsub/KafkaProcessor.java
@@ -14,31 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.processors.kafka.pubsub;
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
-
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
-
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.processor.Relationship;
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
+public class KafkaProcessor {
+ /**
+ * NIFI-11259 provide ability to reference package-private references.
+ */
+ public static final PropertyDescriptor OUTPUT_STRATEGY = ConsumeKafkaRecord_2_6.OUTPUT_STRATEGY;
+ public static final PropertyDescriptor KEY_FORMAT = ConsumeKafkaRecord_2_6.KEY_FORMAT;
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+ public static final Relationship REL_SUCCESS = ConsumeKafkaRecord_2_6.REL_SUCCESS;
+ public static final Relationship REL_PARSE_FAILURE = ConsumeKafkaRecord_2_6.REL_PARSE_FAILURE;
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/resources/org/apache/nifi/processors/kafka/publish/ff.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/resources/org/apache/nifi/processors/kafka/publish/ff.json
new file mode 100644
index 0000000000..98e6b3bd19
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/resources/org/apache/nifi/processors/kafka/publish/ff.json
@@ -0,0 +1 @@
+[{"id":1,"name":"A"},{"id":2,"name":"B"},{"id":3,"name":"C"}]
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/resources/simplelogger.properties b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/resources/simplelogger.properties
new file mode 100644
index 0000000000..f8314c204a
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-integration/src/test/resources/simplelogger.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z
+
+org.slf4j.simpleLogger.log.org.apache.nifi=INFO
+org.slf4j.simpleLogger.log=INFO
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
index d9e7c009ee..86717694db 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
@@ -217,7 +217,7 @@ public abstract class ConsumerLease implements Closeable, ConsumerRebalanceListe
* This behavior has been fixed via Kafka KIP-62 and available from Kafka client 0.10.1.0.
*/
try {
- final ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(Duration.ofMillis(10));
+ final ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(Duration.ofMillis(10L));
lastPollEmpty = records.count() == 0;
processRecords(records);
} catch (final ProcessException pe) {
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/README.md b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/README.md
new file mode 100644
index 0000000000..e709664746
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/README.md
@@ -0,0 +1,12 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership.
+
+ The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+ See the License for the specific language governing permissions and limitations under the License.
+-->
+
+# ConsumeKafkaIT
+- When additional test cases were added to `ConsumeKafkaIT`, intermittent test failures were observed. Experimenting with processor parameters yielded a learning that, in the test containers test environment, reuse of consumer group ID was confusing Kafka client. Use of fresh group ID for each test case led to repeatable successful test runs.
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/pom.xml
new file mode 100644
index 0000000000..d1d86b42b5
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/pom.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-3-integration</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-schema-registry-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kerberos-user-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-services</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>kafka</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-service</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-processors</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractConsumeKafkaIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractConsumeKafkaIT.java
new file mode 100644
index 0000000000..8f60c5dcf4
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractConsumeKafkaIT.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.serialization.StringSerializer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public abstract class AbstractConsumeKafkaIT extends AbstractKafkaBaseIT {
+
+ protected Properties getKafkaProducerProperties() {
+ final Properties properties = new Properties();
+ properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());
+ properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ return properties;
+ }
+
+ protected void produceOne(final String topic, final Integer partition,
+ final String key, final String value, final List<Header> headers)
+ throws ExecutionException, InterruptedException {
+ try (final KafkaProducer<String, String> producer = new KafkaProducer<>(getKafkaProducerProperties())) {
+ final ProducerRecord<String, String> record = new ProducerRecord<>(topic, partition, key, value, headers);
+ final Future<RecordMetadata> future = producer.send(record);
+ final RecordMetadata metadata = future.get();
+ assertEquals(topic, metadata.topic());
+ assertTrue(metadata.hasOffset());
+ assertEquals(0L, metadata.offset());
+ }
+ }
+
+ protected void produce(final String topic, final Collection<ProducerRecord<String, String>> records)
+ throws ExecutionException, InterruptedException {
+ final Collection<RecordMetadata> metadatas = new ArrayList<>();
+ try (final KafkaProducer<String, String> producer = new KafkaProducer<>(getKafkaProducerProperties())) {
+ final Collection<Future<RecordMetadata>> futures = records.stream().map(producer::send).collect(Collectors.toList());
+ for (final Future<RecordMetadata> future : futures) {
+ metadatas.add(future.get());
+ }
+ assertEquals(records.size(), futures.size());
+ assertEquals(futures.size(), metadatas.size());
+ }
+ for (RecordMetadata metadata : metadatas) {
+ assertEquals(topic, metadata.topic());
+ assertTrue(metadata.hasOffset());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractKafkaBaseIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractKafkaBaseIT.java
new file mode 100644
index 0000000000..743c9d0b00
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractKafkaBaseIT.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.nifi.json.JsonRecordSetWriter;
+import org.apache.nifi.json.JsonTreeReader;
+import org.apache.nifi.kafka.service.Kafka3ConnectionService;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.util.TestRunner;
+import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.Properties;
+
+public abstract class AbstractKafkaBaseIT {
+
+ protected static final String IMAGE_NAME = "confluentinc/cp-kafka:7.6.1"; // April 2024
+
+ protected static final long TIMESTAMP = System.currentTimeMillis();
+
+ protected static final String CONNECTION_SERVICE_ID = Kafka3ConnectionService.class.getSimpleName();
+
+ protected static final Duration DURATION_POLL = Duration.ofMillis(1000L);
+
+ protected static final KafkaContainer kafkaContainer;
+
+ // NIFI-11259 - single testcontainers Kafka instance needed for all module integration tests
+ static {
+ kafkaContainer = new KafkaContainer(DockerImageName.parse(IMAGE_NAME));
+ kafkaContainer.start();
+ }
+
+ protected static ObjectMapper objectMapper;
+
+ @BeforeAll
+ protected static void beforeAll() {
+ objectMapper = new ObjectMapper();
+ }
+
+ protected String addKafkaConnectionService(final TestRunner runner) throws InitializationException {
+ final KafkaConnectionService connectionService = new Kafka3ConnectionService();
+ runner.addControllerService(CONNECTION_SERVICE_ID, connectionService);
+ runner.setProperty(connectionService, Kafka3ConnectionService.BOOTSTRAP_SERVERS, kafkaContainer.getBootstrapServers());
+ runner.setProperty(connectionService, Kafka3ConnectionService.MAX_POLL_RECORDS, "1000");
+ runner.enableControllerService(connectionService);
+ return CONNECTION_SERVICE_ID;
+ }
+
+ protected String addRecordReaderService(final TestRunner runner) throws InitializationException {
+ final String readerId = ConsumeKafka.RECORD_READER.getName();
+ final RecordReaderFactory readerService = new JsonTreeReader();
+ runner.addControllerService(readerId, readerService);
+ runner.enableControllerService(readerService);
+ runner.setProperty(readerId, readerId);
+ return readerId;
+ }
+
+ protected String addRecordWriterService(final TestRunner runner) throws InitializationException {
+ final String writerId = ConsumeKafka.RECORD_WRITER.getName();
+ final RecordSetWriterFactory writerService = new JsonRecordSetWriter();
+ runner.addControllerService(writerId, writerService);
+ runner.enableControllerService(writerService);
+ runner.setProperty(writerId, writerId);
+ return writerId;
+ }
+
+ protected String addRecordKeyReaderService(final TestRunner runner) throws InitializationException {
+ final String readerId = ConsumeKafka.KEY_RECORD_READER.getName();
+ final RecordReaderFactory readerService = new JsonTreeReader();
+ runner.addControllerService(readerId, readerService);
+ runner.enableControllerService(readerService);
+ runner.setProperty(readerId, readerId);
+ return readerId;
+ }
+
+ protected String addRecordKeyWriterService(final TestRunner runner) throws InitializationException {
+ final String writerId = PublishKafka.RECORD_KEY_WRITER.getName();
+ final RecordSetWriterFactory writerService = new JsonRecordSetWriter();
+ runner.addControllerService(writerId, writerService);
+ runner.enableControllerService(writerService);
+ runner.setProperty(writerId, writerId);
+ return writerId;
+ }
+
+ protected Properties getKafkaConsumerProperties() {
+ return getKafkaConsumerProperties(kafkaContainer.getBootstrapServers(), "my-group", true);
+ }
+
+ protected Properties getKafkaConsumerProperties(
+ final String bootstrapServers, final String group, final boolean enableAutoCommit) {
+ final Properties properties = new Properties();
+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, group);
+ properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, Boolean.toString(enableAutoCommit));
+ properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
+ properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ return properties;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractPublishKafkaIT.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractPublishKafkaIT.java
index 15aa16ed76..a4462d8b95 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/AbstractPublishKafkaIT.java
@@ -14,31 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.kafka.processors;
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
-
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
-
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
-
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
-
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+public abstract class AbstractPublishKafkaIT extends AbstractKafkaBaseIT {
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaDemarcatorIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaDemarcatorIT.java
new file mode 100644
index 0000000000..f81021b84f
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaDemarcatorIT.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ConsumeKafkaDemarcatorIT extends AbstractConsumeKafkaIT {
+
+ private static final String RECORD_VALUE = "recordA,recordB,recordC,recordD,recordE,recordF";
+ private static final int EXPECTED_TOKENS = 6;
+
+ private static final int FIRST_PARTITION = 0;
+
+ private static final long FIRST_OFFSET = 0;
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue());
+ }
+
+ /**
+ * This configuration amalgamates multiple kafka messages into a single FlowFile.
+ */
+ @Test
+ void testConsumeDemarcated() throws InterruptedException, ExecutionException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.DEMARCATOR);
+
+ runner.setProperty(ConsumeKafka.MESSAGE_DEMARCATOR, "|");
+ runner.setProperty(ConsumeKafka.SEPARATE_BY_KEY, Boolean.FALSE.toString());
+ runner.run(1, false, true);
+
+ final Collection<ProducerRecord<String, String>> records = new ArrayList<>();
+ final String[] values = RECORD_VALUE.split(",");
+ for (String value : values) {
+ records.add(new ProducerRecord<>(topic, value));
+ }
+ produce(topic, records);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+
+ final Iterator<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).iterator();
+ assertTrue(flowFiles.hasNext());
+ final MockFlowFile flowFile = flowFiles.next();
+ flowFile.assertContentEquals(RECORD_VALUE.replaceAll(",", "|"));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(FIRST_PARTITION));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(FIRST_OFFSET));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(EXPECTED_TOKENS - 1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(EXPECTED_TOKENS));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+ }
+
+ /**
+ * This configuration amalgamates multiple kafka messages into FlowFiles, keyed by the provided Kafka message key.
+ */
+ @Test
+ void testConsumeDemarcatedSeparateByKey() throws InterruptedException, ExecutionException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.MESSAGE_DEMARCATOR, "|");
+ runner.setProperty(ConsumeKafka.SEPARATE_BY_KEY, Boolean.TRUE.toString());
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.DEMARCATOR);
+ runner.run(1, false, true);
+
+ final Collection<ProducerRecord<String, String>> records = new ArrayList<>();
+ final String[] values = RECORD_VALUE.split(",");
+ boolean key = false;
+ for (String value : values) {
+ final List<Header> headers = new ArrayList<>();
+ records.add(new ProducerRecord<>(topic, null, Boolean.toString(key), value, headers));
+ key = !key;
+ }
+ produce(topic, records);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 2); // key=false, key=true
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship("success");
+ assertEquals(2, flowFiles.size());
+ for (MockFlowFile flowFile : flowFiles) {
+ if (Boolean.TRUE.toString().equals(flowFile.getAttribute(KafkaFlowFileAttribute.KAFKA_KEY))) {
+ assertEquals("recordB|recordD|recordF", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(EXPECTED_TOKENS - 1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(EXPECTED_TOKENS / 2));
+ } else if (Boolean.FALSE.toString().equals(flowFile.getAttribute(KafkaFlowFileAttribute.KAFKA_KEY))) {
+ assertEquals("recordA|recordC|recordE", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(EXPECTED_TOKENS - 2));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(EXPECTED_TOKENS / 2));
+ } else {
+ Assertions.fail("expected KafkaFlowFileAttribute.KAFKA_KEY");
+ }
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(0));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+ }
+ }
+
+ /**
+ * This configuration amalgamates multiple kafka messages into FlowFiles, keyed by a significant Kafka header.
+ */
+ @Test
+ void testConsumeDemarcatedSeparateByHeader() throws InterruptedException, ExecutionException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.MESSAGE_DEMARCATOR, "|");
+ runner.setProperty(ConsumeKafka.HEADER_NAME_PATTERN, "A.*");
+
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.DEMARCATOR);
+ runner.run(1, false, true);
+
+ final int expectedGroupCount = 3;
+ final Collection<ProducerRecord<String, String>> records = new ArrayList<>();
+ final String[] values = RECORD_VALUE.split(",");
+ int i = 0;
+ for (String value : values) {
+ final String string = String.format("A%d", (++i % expectedGroupCount));
+ final Header header = new RecordHeader(string, string.getBytes(StandardCharsets.UTF_8));
+ records.add(new ProducerRecord<>(topic, null, (String) null, value, Collections.singletonList(header)));
+ }
+ produce(topic, records);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", expectedGroupCount); // header=A1, header=A2, header=A0
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship("success");
+ assertEquals(expectedGroupCount, flowFiles.size());
+ for (MockFlowFile flowFile : flowFiles) {
+
+ if ("A1".equals(flowFile.getAttribute("A1"))) {
+ assertEquals("recordA|recordD", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(0));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(3));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(2));
+ } else if ("A2".equals(flowFile.getAttribute("A2"))) {
+ assertEquals("recordB|recordE", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(1));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(4));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(2));
+ } else if ("A0".equals(flowFile.getAttribute("A0"))) {
+ assertEquals("recordC|recordF", flowFile.getContent());
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(2));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, Long.toString(5));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_COUNT, Long.toString(2));
+ } else {
+ Assertions.fail("expected KafkaFlowFileAttribute");
+ }
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(0));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaIT.java
new file mode 100644
index 0000000000..2e49ef772d
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaIT.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ConsumeKafkaIT extends AbstractConsumeKafkaIT {
+
+ private static final String RECORD_VALUE = ProducerRecord.class.getSimpleName();
+
+ private static final int FIRST_PARTITION = 0;
+
+ private static final long FIRST_OFFSET = 0;
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue());
+ }
+
+ @Test
+ void testProcessingStrategyFlowFileNoRecords() {
+ final String topic = UUID.randomUUID().toString();
+ // reuse of group ID causes issues in test containers test environment
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.FLOW_FILE.getValue());
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ConsumeKafka.SUCCESS, 0);
+ }
+
+ @Test
+ void testProcessingStrategyFlowFileOneRecord() throws InterruptedException, ExecutionException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.FLOW_FILE.getValue());
+ runner.setProperty(ConsumeKafka.HEADER_NAME_PATTERN, "b*");
+
+ runner.run(1, false, true);
+
+ final List<Header> headers = Arrays.asList(
+ new RecordHeader("aaa", "value".getBytes(StandardCharsets.UTF_8)),
+ new RecordHeader("bbb", "value".getBytes(StandardCharsets.UTF_8)),
+ new RecordHeader("ccc", "value".getBytes(StandardCharsets.UTF_8)));
+ produceOne(topic, 0, null, RECORD_VALUE, headers);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+
+ final Iterator<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).iterator();
+ assertTrue(flowFiles.hasNext());
+
+ final MockFlowFile flowFile = flowFiles.next();
+ flowFile.assertContentEquals(RECORD_VALUE);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(FIRST_PARTITION));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(FIRST_OFFSET));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_HEADER_COUNT, "3");
+ flowFile.assertAttributeEquals("bbb", "value");
+ flowFile.assertAttributeNotExists("aaa");
+ flowFile.assertAttributeNotExists("ccc");
+ }
+
+ /**
+ * Test ability to specify a topic regular expression to query for messages.
+ */
+ @Test
+ public void testTopicPattern() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ final String topicTestCase = topic + "-B";
+ final String topicPattern = topic + ".*";
+
+ // on use of "pattern" subscription, things seem to work better when topic exists prior to the subscribe event
+ produceOne(topicTestCase, 0, null, RECORD_VALUE, null);
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topicPattern);
+ runner.setProperty(ConsumeKafka.TOPIC_FORMAT, ConsumeKafka.TOPIC_PATTERN);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.FLOW_FILE.getValue());
+ runner.run(1, false, true);
+
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+ }
+
+ /**
+ * Test ability to specify a topic regular expression to query for messages.
+ */
+ @Test
+ public void testTopicNames() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ final String topicTestCase = topic + "-C";
+ final String topicNames = topic + "-D," + topicTestCase;
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topicNames);
+ runner.setProperty(ConsumeKafka.TOPIC_FORMAT, ConsumeKafka.TOPIC_NAME);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.FLOW_FILE.getValue());
+ runner.run(1, false, true);
+
+ produceOne(topicTestCase, 0, null, RECORD_VALUE, null);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+ runner.assertTransferCount("success", 1);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaKeyAttributeIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaKeyAttributeIT.java
new file mode 100644
index 0000000000..da31d19b09
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaKeyAttributeIT.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ConsumeKafkaKeyAttributeIT extends AbstractConsumeKafkaIT {
+
+ private static final String RECORD_KEY = ConsumeKafkaKeyAttributeIT.class.getSimpleName();
+ private static final String RECORD_VALUE = ConsumeKafkaKeyAttributeIT.class.getName();
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue());
+ }
+
+ /**
+ * Given a processor property and an input, verify the output after the Kafka round-trip.
+ */
+ public static Stream<Arguments> permutations() {
+ return Stream.of(
+ Arguments.arguments(KeyEncoding.UTF8, RECORD_KEY, RECORD_KEY),
+ Arguments.arguments(KeyEncoding.HEX, RECORD_KEY, Hex.encodeHexString(RECORD_KEY.getBytes(StandardCharsets.UTF_8))),
+ Arguments.arguments(KeyEncoding.DO_NOT_ADD, RECORD_KEY, null),
+ Arguments.arguments(KeyEncoding.UTF8, null, null)
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("permutations")
+ void testKeyAttributeEncoding(final KeyEncoding keyEncoding, final String kafkaMessageKey, final String expectedValue)
+ throws InterruptedException, ExecutionException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.FLOW_FILE.getValue());
+ runner.setProperty(ConsumeKafka.KEY_ATTRIBUTE_ENCODING, keyEncoding);
+
+ runner.run(1, false, true);
+ final List<Header> headers = Collections.emptyList();
+ produceOne(topic, 0, kafkaMessageKey, RECORD_VALUE, headers);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+ runner.run(1, true, false);
+
+ final Iterator<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).iterator();
+ assertTrue(flowFiles.hasNext());
+ final MockFlowFile flowFile = flowFiles.next();
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_KEY, expectedValue);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaOffsetsIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaOffsetsIT.java
new file mode 100644
index 0000000000..4c4307226e
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaOffsetsIT.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+class ConsumeKafkaOffsetsIT extends AbstractConsumeKafkaIT {
+
+ private static final String RECORD_VALUE = "recordA,recordB,recordC";
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue());
+ }
+
+ @Test
+ void testVerifyOffsetsNotCommitted() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ final String[] values = RECORD_VALUE.split(",");
+ produceAndConsume(topic, groupId, values, false);
+
+ final Set<TopicPartition> topicPartitions = Collections.singleton(new TopicPartition(topic, 0));
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(
+ getKafkaConsumerProperties(kafkaContainer.getBootstrapServers(), groupId, false))) {
+ final Map<TopicPartition, OffsetAndMetadata> committedOffsets = consumer.committed(topicPartitions);
+ assertEquals(1, committedOffsets.entrySet().size());
+ Map.Entry<TopicPartition, OffsetAndMetadata> entry = committedOffsets.entrySet().iterator().next();
+ assertEquals(topic, entry.getKey().topic());
+ assertNull(entry.getValue());
+ }
+ }
+
+ @Test
+ void testVerifyOffsetsCommitted() throws ExecutionException, InterruptedException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+ final String[] values = RECORD_VALUE.split(",");
+ produceAndConsume(topic, groupId, values, true);
+
+ final Set<TopicPartition> topicPartitions = Collections.singleton(new TopicPartition(topic, 0));
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(
+ getKafkaConsumerProperties(kafkaContainer.getBootstrapServers(), groupId, false))) {
+ final Map<TopicPartition, OffsetAndMetadata> committedOffsets = consumer.committed(topicPartitions);
+ assertEquals(1, committedOffsets.entrySet().size());
+ Map.Entry<TopicPartition, OffsetAndMetadata> entry = committedOffsets.entrySet().iterator().next();
+ assertEquals(topic, entry.getKey().topic());
+ assertEquals(values.length - 1, entry.getValue().offset());
+ }
+ }
+
+ private void produceAndConsume(final String topic, final String groupId, final String[] values, final boolean commitOffsets)
+ throws ExecutionException, InterruptedException {
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.COMMIT_OFFSETS, Boolean.toString(commitOffsets));
+
+ runner.run(1, false, true);
+ final Collection<ProducerRecord<String, String>> records = new ArrayList<>();
+ for (String value : values) {
+ records.add(new ProducerRecord<>(topic, null, (String) null, value, Collections.emptyList()));
+ }
+ produce(topic, records);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while ((System.currentTimeMillis() < pollUntil) && (runner.getFlowFilesForRelationship("success").isEmpty())) {
+ runner.run(1, false, false);
+ }
+ runner.assertTransferCount("success", values.length);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordIT.java
new file mode 100644
index 0000000000..897d307ef5
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordIT.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ConsumeKafkaRecordIT extends AbstractConsumeKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ff.json";
+ private static final int TEST_RECORD_COUNT = 3;
+
+ private static final int FIRST_PARTITION = 0;
+
+ private static final long FIRST_OFFSET = 0;
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ }
+
+ @Test
+ void testProcessingStrategyRecordNoRecords() {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD.getValue());
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ConsumeKafka.SUCCESS, 0);
+ }
+
+ @Test
+ void testProcessingStrategyFlowFileOneRecord() throws InterruptedException, ExecutionException, IOException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD.getValue());
+ runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue());
+ runner.setProperty(ConsumeKafka.HEADER_NAME_PATTERN, "a*");
+
+ runner.run(1, false, true);
+
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ final String flowFileString = new String(bytesFlowFile, StandardCharsets.UTF_8).trim();
+
+ final List<Header> headers = Arrays.asList(
+ new RecordHeader("aaa", "value".getBytes(StandardCharsets.UTF_8)),
+ new RecordHeader("bbb", "value".getBytes(StandardCharsets.UTF_8)),
+ new RecordHeader("ccc", "value".getBytes(StandardCharsets.UTF_8)));
+ produceOne(topic, 0, null, flowFileString, headers);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+
+ runner.run(1, true, false);
+
+ final List<MockFlowFile> flowFilesForRelationship = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS);
+ assertEquals(1, flowFilesForRelationship.size());
+ final Iterator<MockFlowFile> flowFiles = flowFilesForRelationship.iterator();
+ assertTrue(flowFiles.hasNext());
+
+ final MockFlowFile flowFile = flowFiles.next();
+ flowFile.assertContentEquals(flowFileString);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_TOPIC, topic);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(FIRST_PARTITION));
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(FIRST_OFFSET));
+ flowFile.assertAttributeExists(KafkaFlowFileAttribute.KAFKA_TIMESTAMP);
+ flowFile.assertAttributeEquals(KafkaFlowFileAttribute.KAFKA_HEADER_COUNT, "3");
+ flowFile.assertAttributeEquals("record.count", Long.toString(TEST_RECORD_COUNT));
+ flowFile.assertAttributeEquals("aaa", "value");
+ flowFile.assertAttributeNotExists("bbb");
+ flowFile.assertAttributeNotExists("ccc");
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+
+ // [{"id":1,"name":"A"},{"id":2,"name":"B"},{"id":3,"name":"C"}]
+ final JsonNode jsonNodeTree = objectMapper.readTree(flowFile.getContent());
+ assertInstanceOf(ArrayNode.class, jsonNodeTree);
+ final ArrayNode arrayNode = (ArrayNode) jsonNodeTree;
+ final Iterator<JsonNode> elements = arrayNode.elements();
+ assertEquals(TEST_RECORD_COUNT, arrayNode.size());
+ while (elements.hasNext()) {
+ final JsonNode jsonNode = elements.next();
+ assertTrue(Arrays.asList(1, 2, 3).contains(jsonNode.get("id").asInt()));
+ assertTrue(Arrays.asList("A", "B", "C").contains(jsonNode.get("name").asText()));
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaServiceLifecycleIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaServiceLifecycleIT.java
new file mode 100644
index 0000000000..ef9a6e36fc
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaServiceLifecycleIT.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+class ConsumeKafkaServiceLifecycleIT extends AbstractConsumeKafkaIT {
+
+ private static final String CONSUMER_GROUP_ID = ConsumeKafkaIT.class.getName();
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ runner.setProperty(ConsumeKafka.GROUP_ID, CONSUMER_GROUP_ID);
+ }
+
+ /**
+ * Lifecycle of consumer service should be controlled in controller service. Processor should
+ * avoid modifying controller service state.
+ */
+ @Test
+ void testScheduleProcessorMultipleTimes() {
+ runner.setProperty(ConsumeKafka.TOPICS, UUID.randomUUID().toString());
+ runner.run();
+ runner.run();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaWrapperRecordIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaWrapperRecordIT.java
new file mode 100644
index 0000000000..fc6f3c635a
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaWrapperRecordIT.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.KeyFormat;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ConsumeKafkaWrapperRecordIT extends AbstractConsumeKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ff.json";
+ private static final int TEST_RECORD_COUNT = 3;
+ private static final String MESSAGE_KEY = "{\"id\": 0,\"name\": \"K\"}";
+
+ private static final int FIRST_PARTITION = 0;
+ private static final long FIRST_OFFSET = 0;
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(ConsumeKafka.class);
+ addKafkaConnectionService(runner);
+ runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ addRecordKeyReaderService(runner);
+ }
+
+ static abstract class Verifier {
+ abstract void verify(final JsonNode jsonNode);
+ }
+
+ static class RecordVerifier extends Verifier {
+ void verify(final JsonNode jsonNode) {
+ final ObjectNode key = assertInstanceOf(ObjectNode.class, jsonNode);
+ assertEquals(2, key.size());
+ assertEquals(0, key.get("id").asInt());
+ assertEquals("K", key.get("name").asText());
+ }
+ }
+
+ static class StringVerifier extends Verifier {
+ void verify(final JsonNode jsonNode) {
+ final TextNode key = assertInstanceOf(TextNode.class, jsonNode);
+ assertEquals(MESSAGE_KEY, key.asText());
+ }
+ }
+
+ static class ByteArrayVerifier extends Verifier {
+ void verify(final JsonNode jsonNode) {
+ final ArrayNode key = assertInstanceOf(ArrayNode.class, jsonNode);
+ assertEquals(MESSAGE_KEY.length(), key.size());
+ for (int i = 0; (i < MESSAGE_KEY.length()); ++i) {
+ assertEquals(MESSAGE_KEY.charAt(i), key.get(i).asInt());
+ }
+ }
+ }
+
+ public static Stream<Arguments> permutationsKeyFormat() {
+ return Stream.of(
+ Arguments.arguments(KeyFormat.RECORD, new RecordVerifier()),
+ Arguments.arguments(KeyFormat.STRING, new StringVerifier()),
+ Arguments.arguments(KeyFormat.BYTE_ARRAY, new ByteArrayVerifier())
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("permutationsKeyFormat")
+ void testWrapperRecord(final KeyFormat keyFormat, final Verifier verifier)
+ throws InterruptedException, ExecutionException, IOException {
+ final String topic = UUID.randomUUID().toString();
+ final String groupId = topic.substring(0, topic.indexOf("-"));
+
+ runner.setProperty(ConsumeKafka.GROUP_ID, groupId);
+ runner.setProperty(ConsumeKafka.TOPICS, topic);
+ runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD);
+ runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST);
+ runner.setProperty(ConsumeKafka.OUTPUT_STRATEGY, OutputStrategy.USE_WRAPPER);
+ runner.setProperty(ConsumeKafka.KEY_FORMAT, keyFormat);
+ runner.setProperty(ConsumeKafka.HEADER_NAME_PATTERN, "header*");
+
+ runner.run(1, false, true);
+ final String message = new String(IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE))), StandardCharsets.UTF_8);
+ final List<Header> headersPublish = Collections.singletonList(
+ new RecordHeader("header1", "value1".getBytes(StandardCharsets.UTF_8)));
+ produceOne(topic, 0, MESSAGE_KEY, message, headersPublish);
+ final long pollUntil = System.currentTimeMillis() + DURATION_POLL.toMillis();
+ while (System.currentTimeMillis() < pollUntil) {
+ runner.run(1, false, false);
+ }
+ runner.run(1, true, false);
+
+ runner.assertAllFlowFilesTransferred(ConsumeKafka.SUCCESS, 1);
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS);
+ assertEquals(1, flowFiles.size());
+ final MockFlowFile flowFile = flowFiles.getFirst();
+ runner.getLogger().trace(flowFile.getContent());
+ flowFile.assertAttributeEquals("record.count", Long.toString(TEST_RECORD_COUNT));
+
+ final ObjectMapper objectMapper = new ObjectMapper();
+ final JsonNode jsonNodeTree = objectMapper.readTree(flowFile.getContent());
+ final ArrayNode arrayNode = assertInstanceOf(ArrayNode.class, jsonNodeTree);
+
+ final Iterator<JsonNode> elements = arrayNode.elements();
+ assertEquals(TEST_RECORD_COUNT, arrayNode.size());
+ while (elements.hasNext()) {
+ final ObjectNode wrapper = assertInstanceOf(ObjectNode.class, elements.next());
+
+ verifier.verify(wrapper.get("key"));
+
+ final ObjectNode value = assertInstanceOf(ObjectNode.class, wrapper.get("value"));
+ assertTrue(Arrays.asList(1, 2, 3).contains(value.get("id").asInt()));
+ assertTrue(Arrays.asList("A", "B", "C").contains(value.get("name").asText()));
+
+ final ObjectNode headers = assertInstanceOf(ObjectNode.class, wrapper.get("headers"));
+ assertEquals(1, headers.size());
+ assertEquals("value1", headers.get("header1").asText());
+
+ final ObjectNode metadata = assertInstanceOf(ObjectNode.class, wrapper.get("metadata"));
+ assertEquals(topic, metadata.get("topic").asText());
+ assertEquals(FIRST_PARTITION, metadata.get("partition").asInt());
+ assertEquals(FIRST_OFFSET, metadata.get("offset").asInt());
+ assertTrue(metadata.get("timestamp").isIntegralNumber());
+ }
+
+ final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(1, provenanceEvents.size());
+ final ProvenanceEventRecord provenanceEvent = provenanceEvents.getFirst();
+ assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAcksBestEffortIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAcksBestEffortIT.java
new file mode 100644
index 0000000000..fc23652f17
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAcksBestEffortIT.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.nifi.kafka.processors.producer.config.DeliveryGuarantee;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaAcksBestEffortIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ /**
+ * Leaving this here as a placeholder. Need a strategy to handle invalid config at initialization, which throws
+ * on construction:
+ * <ul>
+ * <li>declare exception in interface?</li>
+ * <li>unchecked exception?</li>
+ * <li>reorganize code to avoid Kafka object instantiation in ctor?</li>
+ * </ul>
+ */
+ @Test
+ @Disabled(value = "need a strategy to handle invalid config")
+ public void test_1_KafkaTestContainerProduceOneFail() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.DELIVERY_GUARANTEE, DeliveryGuarantee.DELIVERY_BEST_EFFORT); // invalid if transactionality=true (default)
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.run();
+ }
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, Boolean.FALSE.toString());
+ runner.setProperty(PublishKafka.DELIVERY_GUARANTEE, DeliveryGuarantee.DELIVERY_BEST_EFFORT);
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAcksOneNodeIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAcksOneNodeIT.java
new file mode 100644
index 0000000000..a88c160d41
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAcksOneNodeIT.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.nifi.kafka.processors.producer.config.DeliveryGuarantee;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaAcksOneNodeIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, Boolean.FALSE.toString());
+ runner.setProperty(PublishKafka.DELIVERY_GUARANTEE, DeliveryGuarantee.DELIVERY_ONE_NODE);
+
+ runner.enqueue(TEST_RECORD_VALUE);
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAuthSaslPlaintextIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAuthSaslPlaintextIT.java
new file mode 100644
index 0000000000..21a3f4cbc5
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaAuthSaslPlaintextIT.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.config.SaslConfigs;
+import org.apache.kafka.common.security.plain.PlainLoginModule;
+import org.apache.kafka.common.security.plain.internals.PlainSaslServer;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.nifi.kafka.service.Kafka3ConnectionService;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.kafka.shared.property.SaslMechanism;
+import org.apache.nifi.kafka.shared.property.SecurityProtocol;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@Disabled("circle back to this")
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaAuthSaslPlaintextIT {
+ private static final String SERVICE_ID = Kafka3ConnectionService.class.getSimpleName();
+
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ private static final String USERNAME = "nifi";
+ private static final String PASSWORD = UUID.randomUUID().toString();
+
+ private static KafkaContainer kafka;
+
+ @BeforeAll
+ static void beforeAll() {
+ kafka = new KafkaContainer(DockerImageName.parse(AbstractKafkaBaseIT.IMAGE_NAME))
+ .withEnv(getEnvironmentSaslPlaintext());
+ kafka.start();
+ }
+
+ /**
+ * Environment to be provided to docker container to enable SASL authentication.
+ * <p>
+ * Disable this test for now:
+ * <ul>
+ * <li><a href="https://github.com/testcontainers/testcontainers-java/issues/3899">Kafka SASL mechanism</a></li>
+ * <li><a href="https://github.com/testcontainers/testcontainers-java/issues/6423">Kafka SASL mechanism</a></li>
+ * </ul>
+ */
+ private static Map<String, String> getEnvironmentSaslPlaintext() {
+ final Map<String, String> environment = new LinkedHashMap<>();
+ environment.put("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "BROKER:PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT");
+ environment.put("KAFKA_LISTENER_NAME_PLAINTEXT_SASL_ENABLED_MECHANISMS", "PLAIN");
+ environment.put("KAFKA_LISTENER_NAME_PLAINTEXT_PLAIN_SASL_JAAS_CONFIG", String.format(
+ "%s required user_%s=\"%s\";", PlainLoginModule.class.getName(), USERNAME, PASSWORD));
+ return environment;
+ }
+
+ @AfterAll
+ static void afterAll() {
+ kafka.stop();
+ }
+
+ protected void addKafkaConnectionService(final TestRunner runner) throws InitializationException {
+ final Map<String, String> connectionServiceProps = new HashMap<>();
+ connectionServiceProps.put(Kafka3ConnectionService.BOOTSTRAP_SERVERS.getName(), kafka.getBootstrapServers());
+ connectionServiceProps.put(Kafka3ConnectionService.SECURITY_PROTOCOL.getName(), SecurityProtocol.SASL_PLAINTEXT.name());
+ connectionServiceProps.put(Kafka3ConnectionService.SASL_MECHANISM.getName(), SaslMechanism.PLAIN.name());
+ connectionServiceProps.put(Kafka3ConnectionService.SASL_USERNAME.getName(), USERNAME);
+ connectionServiceProps.put(Kafka3ConnectionService.SASL_PASSWORD.getName(), PASSWORD);
+
+ final KafkaConnectionService connectionService = new Kafka3ConnectionService();
+ runner.addControllerService(SERVICE_ID, connectionService, connectionServiceProps);
+ runner.enableControllerService(connectionService);
+ }
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ addKafkaConnectionService(runner);
+
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, SERVICE_ID);
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(AbstractPublishKafkaIT.DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ }
+ }
+
+ /**
+ * Configure raw Kafka consumer connection to check NiFi publish.
+ */
+ private Properties getKafkaConsumerProperties() {
+ final Properties properties = new Properties();
+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
+ properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, Boolean.TRUE.toString());
+ properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
+ properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+
+ properties.setProperty(SaslConfigs.SASL_MECHANISM, PlainSaslServer.PLAIN_MECHANISM);
+ properties.setProperty(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_PLAINTEXT.name());
+ properties.setProperty(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "%s required username=\"%s\" password=\"%s\";",
+ PlainLoginModule.class.getName(), USERNAME, PASSWORD));
+
+ return properties;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaCompressGZIPIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaCompressGZIPIT.java
new file mode 100644
index 0000000000..5b275ca7b9
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaCompressGZIPIT.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.StringUtils;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaCompressGZIPIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = StringUtils.repeat("value-" + System.currentTimeMillis(), 1000);
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+
+ runner.setProperty(PublishKafka.COMPRESSION_CODEC, "gzip");
+ runner.enqueue(TEST_RECORD_VALUE);
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ /**
+ * Is there a way to programmatically query Kafka to verify the compression?
+ */
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaDelimitedIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaDelimitedIT.java
new file mode 100644
index 0000000000..80de86053d
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaDelimitedIT.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaDelimitedIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ private static final int TEST_RECORD_COUNT = 3;
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.MESSAGE_DEMARCATOR, "xx");
+ runner.setProperty(PublishKafka.ATTRIBUTE_HEADER_PATTERN, "a.*");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+
+ runner.enqueue(StringUtils.repeat(TEST_RECORD_VALUE + "xx", TEST_RECORD_COUNT), attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test2ConsumeMultipleRecords() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(TEST_RECORD_COUNT, records.count());
+ records.forEach(record -> {
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ });
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaFailureStrategyIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaFailureStrategyIT.java
new file mode 100644
index 0000000000..a136a31e02
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaFailureStrategyIT.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.nifi.kafka.shared.property.FailureStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class PublishKafkaFailureStrategyIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ff.not.json";
+
+ @Test
+ public void testProduceRouteToFailure() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.FAILURE_STRATEGY, FailureStrategy.ROUTE_TO_FAILURE);
+
+ // attempt to send a non-json FlowFile to Kafka using record strategy;
+ // this will fail on the record parsing step prior to send; triggering the failure strategy logic
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testProduceRollback() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.FAILURE_STRATEGY, FailureStrategy.ROLLBACK);
+
+ // attempt to send a non-json FlowFile to Kafka using record strategy;
+ // this will fail on the record parsing step prior to send; triggering the failure strategy logic
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ // on rollback, FlowFile is returned to source queue
+ runner.assertTransferCount(PublishKafka.REL_SUCCESS, 0);
+ runner.assertTransferCount(PublishKafka.REL_FAILURE, 0);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaIT.java
new file mode 100644
index 0000000000..40bf041a0a
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaIT.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.ATTRIBUTE_HEADER_PATTERN, "a.*");
+ //runner.setProperty(PublishKafka.USE_TRANSACTIONS, Boolean.FALSE.toString());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+
+ runner.enqueue(TEST_RECORD_VALUE, attributes);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaPartitionStrategyIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaPartitionStrategyIT.java
new file mode 100644
index 0000000000..b7f808e8a3
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaPartitionStrategyIT.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.nifi.kafka.processors.producer.PartitionStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaPartitionStrategyIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.PARTITION_CLASS, PartitionStrategy.EXPRESSION_LANGUAGE_PARTITIONING.getValue());
+ runner.setProperty(PublishKafka.PARTITION, Long.toString(System.currentTimeMillis()));
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ final ConsumerRecord<String, String> record = records.iterator().next();
+ assertNull(record.key());
+ assertEquals(TEST_RECORD_VALUE, record.value());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaRecordIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaRecordIT.java
new file mode 100644
index 0000000000..41fd9dbbc4
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaRecordIT.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaRecordIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ff.json";
+
+ private static final String KEY_ATTRIBUTE_KEY = "keyAttribute";
+ private static final String KEY_ATTRIBUTE_VALUE = "keyAttributeValue";
+
+ private static final int TEST_RECORD_COUNT = 3;
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.KAFKA_KEY, KEY_ATTRIBUTE_KEY);
+ runner.setProperty(PublishKafka.ATTRIBUTE_HEADER_PATTERN, "a.*");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put(KEY_ATTRIBUTE_KEY, KEY_ATTRIBUTE_VALUE);
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test2ConsumeMultipleRecords() throws JsonProcessingException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(TEST_RECORD_COUNT, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ assertEquals(KEY_ATTRIBUTE_VALUE, record.key());
+ assertNotNull(objectMapper.readTree(record.value()));
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaRecordPartialHandlingIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaRecordPartialHandlingIT.java
new file mode 100644
index 0000000000..3aabfea9e8
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaRecordPartialHandlingIT.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.nifi.kafka.shared.property.FailureStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * Test handling of record-based FlowFile, given various NiFi and Kafka configurations. Kafka transactionality allows
+ * records in a FlowFile to succeed or fail as a unit. Non-transactional sends might allow leakage of well-formatted
+ * records at the beginning of a FlowFile with malformed content.
+ */
+public class PublishKafkaRecordPartialHandlingIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ff.partial.json";
+
+ private static Stream<Arguments> publishArguments() {
+ return Stream.of(
+ Arguments.of("no-transactions-route-failure", Boolean.FALSE, FailureStrategy.ROUTE_TO_FAILURE, 0, 1, Boolean.FALSE, 1),
+ Arguments.of("transactions-route-failure", Boolean.TRUE, FailureStrategy.ROUTE_TO_FAILURE, 0, 1, Boolean.FALSE, 0),
+ Arguments.of("no-transactions-rollback", Boolean.FALSE, FailureStrategy.ROLLBACK, 0, 0, Boolean.TRUE, 1), // [1]
+ Arguments.of("transactions-rollback", Boolean.TRUE, FailureStrategy.ROLLBACK, 0, 0, Boolean.TRUE, 0)
+ );
+ }
+
+ // [1] the Kafka client library requires "transactional" mode to be enabled in order to fail all records/KafkaMessages in a given FlowFile
+
+ @ParameterizedTest
+ @MethodSource("publishArguments")
+ public void test1ProduceOneFlowFile(final String label,
+ final Boolean useTransactions,
+ final FailureStrategy failureStrategy,
+ final Integer expectTransferSuccess,
+ final Integer expectTransferFailure,
+ final Boolean expectYield,
+ final Integer expectedKafkaMessageCount) throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ final String topicName = getClass().getName() + "." + label;
+ runner.setProperty(PublishKafka.TOPIC_NAME, topicName);
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, useTransactions.toString());
+ runner.setProperty(PublishKafka.FAILURE_STRATEGY, failureStrategy);
+
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile);
+
+ runner.run(1);
+ runner.assertTransferCount(PublishKafka.REL_SUCCESS, expectTransferSuccess);
+ runner.assertTransferCount(PublishKafka.REL_FAILURE, expectTransferFailure);
+ assertEquals(expectYield, runner.isYieldCalled());
+
+ checkKafkaState(topicName, expectedKafkaMessageCount);
+ }
+
+ public void checkKafkaState(final String topicName, final int expectedKafkaMessageCount) throws JsonProcessingException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(topicName));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(expectedKafkaMessageCount, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ assertNotNull(objectMapper.readTree(record.value()));
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaSSLIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaSSLIT.java
new file mode 100644
index 0000000000..d96f37b3ec
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaSSLIT.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.nifi.kafka.service.Kafka3ConnectionService;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.kafka.shared.property.SecurityProtocol;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.security.util.KeystoreType;
+import org.apache.nifi.ssl.SSLContextService;
+import org.apache.nifi.ssl.StandardRestrictedSSLContextService;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Quick integration test for testing against Kafka cluster defined via
+ * <a href="https://github.com/confluentinc/kafka-images">confluentinc/kafka-images</a>.
+ */
+public class PublishKafkaSSLIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ private static final String BOOTSTRAP_SERVERS = "";
+ private static final String SSL_CONTEXT_SERVICE_PATH = "";
+ private static final String KEYSTORE_PATH = SSL_CONTEXT_SERVICE_PATH + "/kafka.producer.keystore.jks";
+ private static final String TRUSTSTORE_PATH = SSL_CONTEXT_SERVICE_PATH + "/kafka.producer.truststore.jks";
+ private static final String KEYSTORE_PASSWORD = "";
+ private static final String TRUSTSTORE_PASSWORD = "";
+
+ @Test
+ @Disabled("use this to test 'context.yield()' on misconfiguration of KafkaConnectionService; requires running Kafka cluster")
+ public void testKafkaSSLContext() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ private String addKafkaConnectionService(final TestRunner runner) throws InitializationException {
+ final Map<String, String> connectionServiceProps = new HashMap<>();
+ connectionServiceProps.put(Kafka3ConnectionService.BOOTSTRAP_SERVERS.getName(), BOOTSTRAP_SERVERS);
+ connectionServiceProps.put(Kafka3ConnectionService.SSL_CONTEXT_SERVICE.getName(), addSSLContextService(runner));
+ connectionServiceProps.put(Kafka3ConnectionService.SECURITY_PROTOCOL.getName(), SecurityProtocol.SSL.name());
+
+ final String identifier = Kafka3ConnectionService.class.getSimpleName();
+ final KafkaConnectionService connectionService = new Kafka3ConnectionService();
+ runner.addControllerService(identifier, connectionService, connectionServiceProps);
+
+ runner.enableControllerService(connectionService);
+ return identifier;
+ }
+
+ private String addSSLContextService(final TestRunner runner) throws InitializationException {
+ final String identifier = SSLContextService.class.getSimpleName();
+ final SSLContextService service = new StandardRestrictedSSLContextService();
+ runner.addControllerService(identifier, service);
+
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE, KEYSTORE_PATH);
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE_PASSWORD, KEYSTORE_PASSWORD);
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE_TYPE, KeystoreType.JKS.name());
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE, TRUSTSTORE_PATH);
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE_PASSWORD, TRUSTSTORE_PASSWORD);
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE_TYPE, KeystoreType.JKS.name());
+
+ runner.enableControllerService(service);
+ return identifier;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaValueRecordIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaValueRecordIT.java
new file mode 100644
index 0000000000..4c7188c22e
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaValueRecordIT.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaValueRecordIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ffvalue.json";
+
+ private static final String KEY_ATTRIBUTE_KEY = "keyAttribute";
+ private static final String KEY_ATTRIBUTE_VALUE = "keyAttributeValue";
+
+ private static final int TEST_RECORD_COUNT = 3;
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ addRecordKeyWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.KAFKA_KEY, KEY_ATTRIBUTE_KEY);
+ runner.setProperty(PublishKafka.MESSAGE_KEY_FIELD, "address");
+ runner.setProperty(PublishKafka.ATTRIBUTE_HEADER_PATTERN, "a.*");
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_VALUE.name());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put(KEY_ATTRIBUTE_KEY, KEY_ATTRIBUTE_VALUE);
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test2ConsumeMultipleRecords() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(TEST_RECORD_COUNT, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ // kafka record key
+ final ObjectNode kafkaKey = (ObjectNode) objectMapper.readTree(record.key());
+ assertNotNull(kafkaKey);
+ assertEquals("Main", kafkaKey.get("street-name").textValue());
+ assertEquals(5, (kafkaKey.get("street-number").intValue() % 100));
+ // kafka record value (wrapped record)
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertNotEquals(0, kafkaValue.get("id").asInt());
+ assertEquals(1, kafkaValue.get("name").asText().length());
+ assertTrue(kafkaValue.get("address") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaWrapperRecordIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaWrapperRecordIT.java
new file mode 100644
index 0000000000..c1d1dffe8b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/PublishKafkaWrapperRecordIT.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaWrapperRecordIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/ffwrapper.json";
+
+ private static final String KEY_ATTRIBUTE_KEY = "keyAttribute";
+ private static final String KEY_ATTRIBUTE_VALUE = "keyAttributeValue";
+
+ private static final int TEST_RECORD_COUNT = 3;
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ addRecordKeyWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.KAFKA_KEY, KEY_ATTRIBUTE_KEY);
+ runner.setProperty(PublishKafka.MESSAGE_KEY_FIELD, "address");
+ runner.setProperty(PublishKafka.ATTRIBUTE_HEADER_PATTERN, "a.*");
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER.name());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put(KEY_ATTRIBUTE_KEY, KEY_ATTRIBUTE_VALUE);
+ attributes.put("a1", "valueA1");
+ attributes.put("b1", "valueB1");
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeMultipleRecords() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(TEST_RECORD_COUNT, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ final Header header = record.headers().iterator().next();
+ assertEquals("a1", header.key());
+ assertEquals("valueA1", new String(header.value(), StandardCharsets.UTF_8));
+ // kafka record key
+ final ObjectNode kafkaKey = (ObjectNode) objectMapper.readTree(record.key());
+ assertNotNull(kafkaKey);
+ assertEquals("Main", kafkaKey.get("street-name").textValue());
+ assertEquals(5, (kafkaKey.get("street-number").intValue() % 100));
+ // kafka record value (wrapped record)
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertNotEquals(0, kafkaValue.get("id").asInt());
+ assertEquals(1, kafkaValue.get("name").asText().length());
+ assertInstanceOf(ObjectNode.class, kafkaValue.get("address"));
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/PublishKafkaMultipleFFIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/PublishKafkaMultipleFFIT.java
new file mode 100644
index 0000000000..afb5813af0
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/PublishKafkaMultipleFFIT.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish;
+
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.shared.property.FailureStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+public class PublishKafkaMultipleFFIT extends AbstractPublishKafkaIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ public static Stream<Arguments> argumentsTransactionality() {
+ return Stream.of(
+ arguments(Boolean.FALSE),
+ arguments(Boolean.TRUE));
+ }
+
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ public void testKafkaMultipleFlowFilesSuccess(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+
+ final String[] suffixes = {
+ "-A", "-B", "-C"
+ };
+ for (String suffix : suffixes) {
+ runner.enqueue(TEST_RECORD_VALUE + suffix);
+ }
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, suffixes.length);
+ }
+
+ /**
+ * Test NiFi processor failure on attempt to send over-sized Kafka record (NiFi limited).
+ */
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ public void testNiFiFailureTooBig(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+
+ runner.enqueue(new byte[1024 * 1280]); // by default, NiFi maximum is 1MB per record
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_FAILURE, 1);
+ }
+
+ /**
+ * Test Kafka client library failure on attempt to send over-sized Kafka record (Kafka limited).
+ */
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ public void testKafkaFailureTooBig(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+ runner.setProperty(PublishKafka.MAX_REQUEST_SIZE, "2 MB");
+
+ runner.enqueue(new byte[1024 * 1280]); // by default, Kafka maximum is 1MB per record
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_FAILURE, 1);
+ }
+
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ public void testKafkaMultipleFlowFilesPartialFailureRollback(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+ runner.setProperty(PublishKafka.FAILURE_STRATEGY, FailureStrategy.ROLLBACK.getValue());
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.enqueue(new byte[1024 * 1280]); // by default, max 1MB per record
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, 1);
+ }
+
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ public void testKafkaMultipleFlowFilesPartialFailureTransferFailure(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+ runner.setProperty(PublishKafka.FAILURE_STRATEGY, FailureStrategy.ROUTE_TO_FAILURE);
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.enqueue(new byte[1024 * 1280]); // by default, max 1MB per record
+ runner.run(2);
+ runner.assertTransferCount(PublishKafka.REL_SUCCESS, 1);
+ runner.assertTransferCount(PublishKafka.REL_FAILURE, 1);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaContentX1IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaContentX1IT.java
new file mode 100644
index 0000000000..f7fba11276
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaContentX1IT.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.additional;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaContentX1IT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/additional/contentX1.json";
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ addRecordKeyWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_VALUE.name());
+ runner.setProperty(PublishKafka.MESSAGE_KEY_FIELD, "account");
+ runner.setProperty(PublishKafka.ATTRIBUTE_HEADER_PATTERN, "attribute.*");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attributeA", "valueA");
+ attributes.put("attributeB", "valueB");
+ attributes.put("otherAttribute", "otherValue");
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test2ConsumeOneRecord() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(2, headers.size());
+ assertTrue(headers.contains(new RecordHeader("attributeA", "valueA".getBytes(StandardCharsets.UTF_8))));
+ assertTrue(headers.contains(new RecordHeader("attributeB", "valueB".getBytes(StandardCharsets.UTF_8))));
+ // kafka record key
+ final ObjectNode kafkaKey = (ObjectNode) objectMapper.readTree(record.key());
+ assertNotNull(kafkaKey);
+ assertEquals("Acme", kafkaKey.get("name").textValue());
+ assertEquals("AC1234", kafkaKey.get("number").textValue());
+ // kafka record value
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertEquals("1234 First Street", kafkaValue.get("address").textValue());
+ assertEquals("12345", kafkaValue.get("zip").textValue());
+ assertTrue(kafkaValue.get("account") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX1IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX1IT.java
new file mode 100644
index 0000000000..12e6973812
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX1IT.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.additional;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaWrapperX1IT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/additional/wrapperX1.json";
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER.name());
+
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test2ConsumeOneRecord() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(2, headers.size());
+ assertTrue(headers.contains(new RecordHeader("accountType", "enterprise".getBytes(StandardCharsets.UTF_8))));
+ assertTrue(headers.contains(new RecordHeader("test", "true".getBytes(StandardCharsets.UTF_8))));
+ // kafka record key
+ assertEquals("Acme Holdings", record.key());
+ // kafka record value
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertEquals("1234 First Street", kafkaValue.get("address").textValue());
+ assertEquals("12345", kafkaValue.get("zip").textValue());
+ assertTrue(kafkaValue.get("account") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX2IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX2IT.java
new file mode 100644
index 0000000000..71299901d0
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX2IT.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.additional;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaWrapperX2IT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/additional/wrapperX2.json";
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+ addRecordKeyWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER.name());
+
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test2ConsumeOneRecord() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record metadata
+ assertEquals(getClass().getName(), record.topic());
+ assertEquals(0, record.partition());
+ assertTrue(record.timestamp() > TIMESTAMP);
+ assertTrue(record.timestamp() < System.currentTimeMillis());
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertTrue(headers.isEmpty());
+ // kafka record key
+ final ObjectNode kafkaKey = (ObjectNode) objectMapper.readTree(record.key());
+ assertNotNull(kafkaKey);
+ assertEquals("Acme Holdings", kafkaKey.get("accountName").textValue());
+ assertEquals("John Doe", kafkaKey.get("accountHolder").textValue());
+ assertEquals("280182830-A009", kafkaKey.get("accountId").textValue());
+ // kafka record value
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertEquals("1234 First Street", kafkaValue.get("address").textValue());
+ assertEquals("12345", kafkaValue.get("zip").textValue());
+ assertTrue(kafkaValue.get("account") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX3IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX3IT.java
new file mode 100644
index 0000000000..00fa62341e
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX3IT.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.additional;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaWrapperX3IT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/additional/wrapperX3.json";
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER.name());
+
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test2ConsumeOneRecord() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record metadata
+ assertEquals(getClass().getName(), record.topic());
+ assertEquals(0, record.partition());
+ assertTrue(record.timestamp() > TIMESTAMP);
+ assertTrue(record.timestamp() < System.currentTimeMillis());
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertTrue(headers.isEmpty());
+ // kafka record key
+ assertEquals("411b0a140b39581341", Hex.encodeHexString(record.key().getBytes(StandardCharsets.UTF_8)));
+ // kafka record value
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertEquals("1234 First Street", kafkaValue.get("address").textValue());
+ assertEquals("12345", kafkaValue.get("zip").textValue());
+ assertTrue(kafkaValue.get("account") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX4IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX4IT.java
new file mode 100644
index 0000000000..d16387d59b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX4IT.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.additional;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaWrapperX4IT extends AbstractPublishKafkaIT {
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/additional/wrapperX4.json";
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER.name());
+
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFile = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test2ConsumeOneRecord() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Collections.singletonList(getClass().getName()));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record metadata
+ assertEquals(getClass().getName(), record.topic());
+ assertEquals(0, record.partition());
+ assertTrue(record.timestamp() > TIMESTAMP);
+ assertTrue(record.timestamp() < System.currentTimeMillis());
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(2, headers.size());
+ assertTrue(headers.contains(new RecordHeader("a", "b".getBytes(StandardCharsets.UTF_8))));
+ assertTrue(headers.contains(new RecordHeader("c", "{\"d\":\"e\"}".getBytes(StandardCharsets.UTF_8))));
+ // kafka record key
+ assertNull(record.key());
+ // kafka record value
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertEquals("1234 First Street", kafkaValue.get("address").textValue());
+ assertEquals("12345", kafkaValue.get("zip").textValue());
+ assertTrue(kafkaValue.get("account") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX5IT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX5IT.java
new file mode 100644
index 0000000000..3de3da9417
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/PublishKafkaWrapperX5IT.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.additional;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.CreateTopicsResult;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.processors.AbstractPublishKafkaIT;
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.processors.producer.wrapper.RecordMetadataStrategy;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class PublishKafkaWrapperX5IT extends AbstractPublishKafkaIT {
+ private static final String TEST_TOPIC = "nifi-" + PublishKafkaWrapperX5IT.class.getName();
+ private static final String OVERRIDE_TOPIC = "topic1-" + PublishKafkaWrapperX5IT.class.getName();
+ private static final Integer TEST_PARTITION = new Random().nextInt(3) - 1;
+ private static final Integer OVERRIDE_PARTITION = 1;
+
+ private static final String TEST_RESOURCE = "org/apache/nifi/kafka/processors/publish/additional/wrapperX5.json";
+
+ @BeforeAll
+ protected static void beforeAll() {
+ AbstractPublishKafkaIT.beforeAll();
+
+ final Properties properties = new Properties();
+ properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());
+ try (final Admin adminClient = Admin.create(properties)) {
+ final int numPartitions = 3;
+ final short replicationFactor = 1;
+ final NewTopic testTopic = new NewTopic(TEST_TOPIC, numPartitions, replicationFactor);
+ final NewTopic overrideTopic = new NewTopic(OVERRIDE_TOPIC, numPartitions, replicationFactor);
+ final CreateTopicsResult topics = adminClient.createTopics(Arrays.asList(testTopic, overrideTopic));
+ final KafkaFuture<Void> testTopicFuture = topics.values().get(TEST_TOPIC);
+ final KafkaFuture<Void> overrideTopicFuture = topics.values().get(OVERRIDE_TOPIC);
+ testTopicFuture.get(1, TimeUnit.SECONDS);
+ overrideTopicFuture.get(1, TimeUnit.SECONDS);
+ } catch (ExecutionException | InterruptedException | TimeoutException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void test1ProduceOneFlowFile() throws InitializationException, IOException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ addRecordReaderService(runner);
+ addRecordWriterService(runner);
+
+ runner.setProperty(PublishKafka.TOPIC_NAME, TEST_TOPIC);
+ runner.setProperty(PublishKafka.PARTITION, Integer.toString(TEST_PARTITION));
+ runner.getLogger().info("partition={}", TEST_PARTITION);
+ runner.setProperty(PublishKafka.PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER.name());
+ runner.setProperty(PublishKafka.RECORD_METADATA_STRATEGY, RecordMetadataStrategy.FROM_RECORD.getValue());
+
+ final Map<String, String> attributes = new HashMap<>();
+ final byte[] bytesFlowFileTemplate = IOUtils.toByteArray(Objects.requireNonNull(
+ getClass().getClassLoader().getResource(TEST_RESOURCE)));
+ final byte[] bytesFlowFile = new String(bytesFlowFileTemplate, StandardCharsets.UTF_8)
+ .replace("topic1", OVERRIDE_TOPIC).getBytes(StandardCharsets.UTF_8);
+ runner.enqueue(bytesFlowFile, attributes);
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred("success", 1);
+ }
+
+ @Test
+ public void test2ConsumeOneRecord() throws IOException {
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(getKafkaConsumerProperties())) {
+ consumer.subscribe(Arrays.asList(TEST_TOPIC, OVERRIDE_TOPIC));
+ final ConsumerRecords<String, String> records = consumer.poll(DURATION_POLL);
+ assertEquals(1, records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ // kafka record metadata
+ assertEquals(OVERRIDE_TOPIC, record.topic());
+ assertEquals(OVERRIDE_PARTITION, record.partition());
+ assertTrue(record.timestamp() > TIMESTAMP);
+ assertTrue(record.timestamp() < System.currentTimeMillis());
+ // kafka record headers
+ final List<Header> headers = Arrays.asList(record.headers().toArray());
+ assertEquals(1, headers.size());
+ assertTrue(headers.contains(new RecordHeader("a", "b".getBytes(StandardCharsets.UTF_8))));
+ // kafka record key
+ assertNull(record.key());
+ // kafka record value
+ final ObjectNode kafkaValue = (ObjectNode) objectMapper.readTree(record.value());
+ assertNotNull(kafkaValue);
+ assertEquals("1234 First Street", kafkaValue.get("address").textValue());
+ assertEquals("12345", kafkaValue.get("zip").textValue());
+ assertTrue(kafkaValue.get("account") instanceof ObjectNode);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/README.md b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/README.md
new file mode 100644
index 0000000000..0ca851be5f
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/additional/README.md
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+= Notes
+Integration tests in this folder derived from examples provided in:
+https://github.com/apache/nifi/blob/main/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/dependency/PublishKafkaMultipleFFIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/dependency/PublishKafkaMultipleFFIT.java
new file mode 100644
index 0000000000..9e91c1d90b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/publish/dependency/PublishKafkaMultipleFFIT.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.publish.dependency;
+
+import org.apache.nifi.kafka.processors.PublishKafka;
+import org.apache.nifi.kafka.service.Kafka3ConnectionService;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.kafka.shared.property.FailureStrategy;
+import org.apache.nifi.kafka.shared.property.SecurityProtocol;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.security.util.KeystoreType;
+import org.apache.nifi.ssl.SSLContextService;
+import org.apache.nifi.ssl.StandardRestrictedSSLContextService;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+/**
+ * Quick integration test for testing against Kafka cluster defined via
+ * <a href="https://github.com/confluentinc/kafka-images">confluentinc/kafka-images</a>.
+ */
+public class PublishKafkaMultipleFFIT {
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ private static final String BOOTSTRAP_SERVERS = "";
+ private static final String SSL_CONTEXT_SERVICE_PATH = "";
+ private static final String KEYSTORE_PATH = SSL_CONTEXT_SERVICE_PATH + "/kafka.producer.keystore.jks";
+ private static final String TRUSTSTORE_PATH = SSL_CONTEXT_SERVICE_PATH + "/kafka.producer.truststore.jks";
+ private static final String KEYSTORE_PASSWORD = "";
+ private static final String TRUSTSTORE_PASSWORD = "";
+
+ public static Stream<Arguments> argumentsTransactionality() {
+ return Stream.of(
+ arguments(Boolean.FALSE),
+ arguments(Boolean.TRUE));
+ }
+
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ @Disabled("use this to test publish of multiple FlowFiles; requires running Kafka cluster")
+ public void testKafkaMultipleFlowFilesSuccess(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+
+ final String[] suffixes = {
+ "-A", "-B", "-C"
+ };
+ for (String suffix : suffixes) {
+ runner.enqueue(TEST_RECORD_VALUE + suffix);
+ }
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_SUCCESS, suffixes.length);
+ }
+
+ /**
+ * Test NiFi processor failure on attempt to send over-sized Kafka record (NiFi limited).
+ */
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ @Disabled("use this to test partial publish failure; requires running Kafka cluster")
+ public void testNiFiFailureTooBig(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+
+ runner.enqueue(new byte[1024 * 1280]); // by default, NiFi maximum is 1MB per record
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_FAILURE, 1);
+ }
+
+ /**
+ * Test Kafka client library failure on attempt to send over-sized Kafka record (Kafka limited).
+ */
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ @Disabled("use this to test partial publish failure; requires running Kafka cluster")
+ public void testKafkaFailureTooBig(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+ runner.setProperty(PublishKafka.MAX_REQUEST_SIZE, "2 MB");
+
+ runner.enqueue(new byte[1024 * 1280]); // by default, Kafka maximum is 1MB per record
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PublishKafka.REL_FAILURE, 1);
+ }
+
+ @ParameterizedTest
+ @MethodSource("argumentsTransactionality")
+ @Disabled("use this to test partial publish failure; requires running Kafka cluster")
+ public void testKafkaMultipleFlowFilesPartialFailure(final Boolean transactionality) throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(PublishKafka.class);
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(PublishKafka.CONNECTION_SERVICE, addKafkaConnectionService(runner));
+ runner.setProperty(PublishKafka.TOPIC_NAME, getClass().getName());
+ runner.setProperty(PublishKafka.TRANSACTIONS_ENABLED, transactionality.toString());
+ runner.setProperty(PublishKafka.FAILURE_STRATEGY, FailureStrategy.ROLLBACK.getValue());
+
+ runner.enqueue(TEST_RECORD_VALUE);
+ runner.enqueue(new byte[1024 * 1280]); // by default, max 1MB per record
+ runner.run();
+ runner.assertTransferCount(PublishKafka.REL_SUCCESS, 0);
+ runner.assertTransferCount(PublishKafka.REL_FAILURE, 0);
+ }
+
+ private String addKafkaConnectionService(final TestRunner runner) throws InitializationException {
+ final Map<String, String> connectionServiceProps = new HashMap<>();
+ connectionServiceProps.put(Kafka3ConnectionService.BOOTSTRAP_SERVERS.getName(), BOOTSTRAP_SERVERS);
+ connectionServiceProps.put(Kafka3ConnectionService.SSL_CONTEXT_SERVICE.getName(), addSSLContextService(runner));
+ connectionServiceProps.put(Kafka3ConnectionService.SECURITY_PROTOCOL.getName(), SecurityProtocol.SSL.name());
+
+ final String identifier = Kafka3ConnectionService.class.getSimpleName();
+ final KafkaConnectionService connectionService = new Kafka3ConnectionService();
+ runner.addControllerService(identifier, connectionService, connectionServiceProps);
+
+ runner.enableControllerService(connectionService);
+ return identifier;
+ }
+
+ private String addSSLContextService(final TestRunner runner) throws InitializationException {
+ final String identifier = SSLContextService.class.getSimpleName();
+ final SSLContextService service = new StandardRestrictedSSLContextService();
+ runner.addControllerService(identifier, service);
+
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE, KEYSTORE_PATH);
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE_PASSWORD, KEYSTORE_PASSWORD);
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE_TYPE, KeystoreType.JKS.name());
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE, TRUSTSTORE_PATH);
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE_PASSWORD, TRUSTSTORE_PASSWORD);
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE_TYPE, KeystoreType.JKS.name());
+
+ runner.enableControllerService(service);
+ return identifier;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/contentX1.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/contentX1.json
new file mode 100644
index 0000000000..f499b3a539
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/contentX1.json
@@ -0,0 +1 @@
+{"address":"1234 First Street","zip":"12345","account":{"name":"Acme","number":"AC1234"}}
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX1.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX1.json
new file mode 100644
index 0000000000..79b64fe9d4
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX1.json
@@ -0,0 +1,15 @@
+{
+ "key": "Acme Holdings",
+ "value": {
+ "address": "1234 First Street",
+ "zip": "12345",
+ "account": {
+ "name": "Acme",
+ "number":"AC1234"
+ }
+ },
+ "headers": {
+ "accountType": "enterprise",
+ "test": "true"
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX2.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX2.json
new file mode 100644
index 0000000000..67021a3b41
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX2.json
@@ -0,0 +1,15 @@
+{
+ "key": {
+ "accountName": "Acme Holdings",
+ "accountHolder": "John Doe",
+ "accountId": "280182830-A009"
+ },
+ "value": {
+ "address": "1234 First Street",
+ "zip": "12345",
+ "account": {
+ "name": "Acme",
+ "number":"AC1234"
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX3.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX3.json
new file mode 100644
index 0000000000..a729302a11
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX3.json
@@ -0,0 +1,14 @@
+{
+ "key": [65, 27, 10, 20, 11, 57, 88, 19, 65],
+ "value": {
+ "address": "1234 First Street",
+ "zip": "12345",
+ "account": {
+ "name": "Acme",
+ "number":"AC1234"
+ }
+ },
+ "otherField": {
+ "a": "b"
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX4.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX4.json
new file mode 100644
index 0000000000..2f89a9e3a2
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX4.json
@@ -0,0 +1,16 @@
+{
+ "value": {
+ "address": "1234 First Street",
+ "zip": "12345",
+ "account": {
+ "name": "Acme",
+ "number": "AC1234"
+ }
+ },
+ "headers": {
+ "a": "b",
+ "c": {
+ "d": "e"
+ }
+ }
+}
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX5.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX5.json
new file mode 100644
index 0000000000..44e686a4ab
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/additional/wrapperX5.json
@@ -0,0 +1,17 @@
+{
+ "value": {
+ "address": "1234 First Street",
+ "zip": "12345",
+ "account": {
+ "name": "Acme",
+ "number": "AC1234"
+ }
+ },
+ "headers": {
+ "a": "b"
+ },
+ "metadata": {
+ "topic": "topic1",
+ "partition": "1"
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.json
new file mode 100644
index 0000000000..98e6b3bd19
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.json
@@ -0,0 +1 @@
+[{"id":1,"name":"A"},{"id":2,"name":"B"},{"id":3,"name":"C"}]
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.not.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.not.json
new file mode 100644
index 0000000000..34230f07b1
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.not.json
@@ -0,0 +1 @@
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.partial.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.partial.json
new file mode 100644
index 0000000000..94f8120a6b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ff.partial.json
@@ -0,0 +1,8 @@
+[
+ {
+ "a": "A",
+ "b": "B"
+ },
+ true,
+ -2106859769
+]
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ffvalue.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ffvalue.json
new file mode 100644
index 0000000000..48af3c91bc
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ffvalue.json
@@ -0,0 +1,26 @@
+[
+ {
+ "id": 1,
+ "name": "A",
+ "address": {
+ "street-number": 105,
+ "street-name": "Main"
+ }
+ },
+ {
+ "id": 2,
+ "name": "B",
+ "address": {
+ "street-number": 205,
+ "street-name": "Main"
+ }
+ },
+ {
+ "id": 3,
+ "name": "C",
+ "address": {
+ "street-number": 305,
+ "street-name": "Main"
+ }
+ }
+]
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ffwrapper.json b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ffwrapper.json
new file mode 100644
index 0000000000..6fcb5f9e61
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/org/apache/nifi/kafka/processors/publish/ffwrapper.json
@@ -0,0 +1,53 @@
+[
+ {
+ "headers": {
+ "a1": "valueA1"
+ },
+ "key": {
+ "street-number": 105,
+ "street-name": "Main"
+ },
+ "value": {
+ "id": 1,
+ "name": "A",
+ "address": {
+ "street-number": 105,
+ "street-name": "Main"
+ }
+ }
+ },
+ {
+ "headers": {
+ "a1": "valueA1"
+ },
+ "key": {
+ "street-number": 205,
+ "street-name": "Main"
+ },
+ "value": {
+ "id": 2,
+ "name": "B",
+ "address": {
+ "street-number": 205,
+ "street-name": "Main"
+ }
+ }
+ },
+ {
+ "headers": {
+ "a1": "valueA1"
+ },
+ "key": {
+ "street-number": 305,
+ "street-name": "Main"
+ },
+ "value": {
+ "id": 3,
+ "name": "C",
+ "address": {
+ "street-number": 305,
+ "street-name": "Main"
+ }
+ }
+ }
+]
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/simplelogger.properties b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/simplelogger.properties
new file mode 100644
index 0000000000..d5fef01a27
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/resources/simplelogger.properties
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z
+
+org.slf4j.simpleLogger.log.org.apache.kafka.clients.producer.ProducerConfig=WARN
+org.slf4j.simpleLogger.log.org.apache.nifi.kafka.processors.PublishKafka=TRACE
+org.slf4j.simpleLogger.log.org.apache.nifi.kafka.service.producer=TRACE
+org.slf4j.simpleLogger.log=INFO
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service-nar/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service-nar/pom.xml
new file mode 100644
index 0000000000..3e88740e5d
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service-nar/pom.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-3-service-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-service</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api-nar</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/pom.xml
new file mode 100644
index 0000000000..4ccb1f1d0a
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/pom.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-3-service</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kerberos-user-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-shared</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-pool2</artifactId>
+ <version>2.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>kafka</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>3.7.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- Included for nifi-utils StringUtils -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
new file mode 100644
index 0000000000..dbc9dca2e3
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service;
+
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.ListTopicsResult;
+import org.apache.kafka.clients.admin.TopicListing;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
+import org.apache.nifi.annotation.behavior.DynamicProperty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.lifecycle.OnDisabled;
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.context.PropertyContext;
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.controller.VerifiableControllerService;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.kafka.service.api.common.ServiceConfiguration;
+import org.apache.nifi.kafka.service.api.consumer.ConsumerConfiguration;
+import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
+import org.apache.nifi.kafka.service.api.producer.KafkaProducerService;
+import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration;
+import org.apache.nifi.kafka.service.consumer.Kafka3ConsumerService;
+import org.apache.nifi.kafka.service.producer.Kafka3ProducerService;
+import org.apache.nifi.kafka.shared.property.SaslMechanism;
+import org.apache.nifi.kafka.shared.property.IsolationLevel;
+import org.apache.nifi.kafka.shared.property.provider.KafkaPropertyProvider;
+import org.apache.nifi.kafka.shared.property.provider.StandardKafkaPropertyProvider;
+import org.apache.nifi.kafka.shared.transaction.TransactionIdSupplier;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.ssl.SSLContextService;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.nifi.components.ConfigVerificationResult.Outcome.FAILED;
+import static org.apache.nifi.components.ConfigVerificationResult.Outcome.SUCCESSFUL;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_KEYSTORE_LOCATION;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_KEYSTORE_PASSWORD;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_KEYSTORE_TYPE;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_KEY_PASSWORD;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_TRUSTSTORE_LOCATION;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_TRUSTSTORE_PASSWORD;
+import static org.apache.nifi.kafka.shared.property.KafkaClientProperty.SSL_TRUSTSTORE_TYPE;
+
+@DynamicProperty(name = "The name of a Kafka configuration property.", value = "The value of a given Kafka configuration property.",
+ description = "These properties will be added on the Kafka configuration after loading any provided configuration properties."
+ + " In the event a dynamic property represents a property that was already set, its value will be ignored and WARN message logged."
+ + " For the list of available Kafka properties please refer to: http://kafka.apache.org/documentation.html#configuration.",
+ expressionLanguageScope = ExpressionLanguageScope.ENVIRONMENT)
+@CapabilityDescription("Provides and manages connections to Kafka Brokers for producer or consumer operations.")
+public class Kafka3ConnectionService extends AbstractControllerService implements KafkaConnectionService, VerifiableControllerService {
+
+ public static final PropertyDescriptor BOOTSTRAP_SERVERS = new PropertyDescriptor.Builder()
+ .name("bootstrap.servers")
+ .displayName("Bootstrap Servers")
+ .description("Comma-separated list of Kafka Bootstrap Servers in the format host:port. Corresponds to Kafka bootstrap.servers property")
+ .required(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .build();
+
+ public static final PropertyDescriptor SECURITY_PROTOCOL = new PropertyDescriptor.Builder()
+ .name("security.protocol")
+ .displayName("Security Protocol")
+ .description("Security protocol used to communicate with brokers. Corresponds to Kafka Client security.protocol property")
+ .required(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .allowableValues(SecurityProtocol.values())
+ .defaultValue(SecurityProtocol.PLAINTEXT.name())
+ .build();
+
+ public static final PropertyDescriptor SASL_MECHANISM = new PropertyDescriptor.Builder()
+ .name("sasl.mechanism")
+ .displayName("SASL Mechanism")
+ .description("SASL mechanism used for authentication. Corresponds to Kafka Client sasl.mechanism property")
+ .required(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .allowableValues(SaslMechanism.getAvailableSaslMechanisms())
+ .defaultValue(SaslMechanism.GSSAPI.getValue())
+ .build();
+
+ public static final PropertyDescriptor SASL_USERNAME = new PropertyDescriptor.Builder()
+ .name("sasl.username")
+ .displayName("SASL Username")
+ .description("Username provided with configured password when using PLAIN or SCRAM SASL Mechanisms")
+ .required(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
+ .dependsOn(
+ SASL_MECHANISM,
+ SaslMechanism.PLAIN.getValue(),
+ SaslMechanism.SCRAM_SHA_256.getValue(),
+ SaslMechanism.SCRAM_SHA_512.getValue()
+ )
+ .build();
+
+ public static final PropertyDescriptor SASL_PASSWORD = new PropertyDescriptor.Builder()
+ .name("sasl.password")
+ .displayName("SASL Password")
+ .description("Password provided with configured username when using PLAIN or SCRAM SASL Mechanisms")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
+ .dependsOn(
+ SASL_MECHANISM,
+ SaslMechanism.PLAIN.getValue(),
+ SaslMechanism.SCRAM_SHA_256.getValue(),
+ SaslMechanism.SCRAM_SHA_512.getValue()
+ )
+ .build();
+
+ public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
+ .name("SSL Context Service")
+ .description("Service supporting SSL communication with Kafka brokers")
+ .required(false)
+ .identifiesControllerService(SSLContextService.class)
+ .build();
+
+ public static final PropertyDescriptor TRANSACTION_ISOLATION_LEVEL = new PropertyDescriptor.Builder()
+ .name("isolation.level")
+ .displayName("Transaction Isolation Level")
+ .description("""
+ Specifies how the service should handle transaction isolation levels when communicating with Kafka.
+ The uncommited option means that messages will be received as soon as they are written to Kafka but will be pulled, even if the producer cancels the transactions.
+ The committed option configures the service to not receive any messages for which the producer's transaction was canceled, but this can result in some latency since the
+ consumer must wait for the producer to finish its entire transaction instead of pulling as the messages become available.
+ Corresponds to Kafka isolation.level property.
+ """)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .allowableValues(IsolationLevel.class)
+ .defaultValue(IsolationLevel.READ_COMMITTED)
+ .required(true)
+ .build();
+
+ public static final PropertyDescriptor MAX_POLL_RECORDS = new PropertyDescriptor.Builder()
+ .name("max.poll.records")
+ .displayName("Max Poll Records")
+ .description("Maximum number of records Kafka should return in a single poll.")
+ .required(true)
+ .defaultValue("10000")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CLIENT_TIMEOUT = new PropertyDescriptor.Builder()
+ .name("default.api.timeout.ms")
+ .displayName("Client Timeout")
+ .description("Default timeout for Kafka client operations. Mapped to Kafka default.api.timeout.ms. The Kafka request.timeout.ms property is derived from half of the configured timeout")
+ .defaultValue("60 s")
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .build();
+
+ public static final PropertyDescriptor METADATA_WAIT_TIME = new PropertyDescriptor.Builder()
+ .name("max.block.ms")
+ .displayName("Max Metadata Wait Time")
+ .description("""
+ The amount of time publisher will wait to obtain metadata or wait for the buffer to flush during the 'send' call before failing the
+ entire 'send' call. Corresponds to Kafka max.block.ms property
+ """)
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
+ .defaultValue("5 s")
+ .build();
+
+ public static final PropertyDescriptor ACK_WAIT_TIME = new PropertyDescriptor.Builder()
+ .name("ack.wait.time")
+ .displayName("Acknowledgment Wait Time")
+ .description("""
+ After sending a message to Kafka, this indicates the amount of time that the service will wait for a response from Kafka.
+ If Kafka does not acknowledge the message within this time period, the service will throw an exception.
+ """)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .required(true)
+ .defaultValue("5 s")
+ .build();
+
+ private static final List<PropertyDescriptor> PROPERTY_DESCRIPTORS = List.of(
+ BOOTSTRAP_SERVERS,
+ SECURITY_PROTOCOL,
+ SASL_MECHANISM,
+ SASL_USERNAME,
+ SASL_PASSWORD,
+ SSL_CONTEXT_SERVICE,
+ TRANSACTION_ISOLATION_LEVEL,
+ MAX_POLL_RECORDS,
+ CLIENT_TIMEOUT,
+ METADATA_WAIT_TIME,
+ ACK_WAIT_TIME
+ );
+
+ private static final Duration VERIFY_TIMEOUT = Duration.ofSeconds(2);
+
+ private static final String CONNECTION_STEP = "Kafka Broker Connection";
+
+ private static final String TOPIC_LISTING_STEP = "Kafka Topic Listing";
+
+ private Properties clientProperties;
+
+ private ServiceConfiguration serviceConfiguration;
+
+ private Kafka3ConsumerService consumerService;
+
+ @OnEnabled
+ public void onEnabled(final ConfigurationContext configurationContext) {
+ clientProperties = getClientProperties(configurationContext);
+ serviceConfiguration = getServiceConfiguration(configurationContext);
+ final Properties consumerProperties = getConsumerProperties(configurationContext, clientProperties);
+ consumerService = new Kafka3ConsumerService(getLogger(), consumerProperties);
+ }
+
+ @OnDisabled
+ public void onDisabled() {
+ if (consumerService == null) {
+ getLogger().warn("Consumer Service not configured");
+ } else {
+ consumerService.close();
+ }
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTY_DESCRIPTORS;
+ }
+
+ @Override
+ public KafkaConsumerService getConsumerService(final ConsumerConfiguration consumerConfiguration) {
+ return consumerService;
+ }
+
+ @Override
+ public KafkaProducerService getProducerService(final ProducerConfiguration producerConfiguration) {
+ final Properties propertiesProducer = new Properties();
+ propertiesProducer.putAll(clientProperties);
+ if (producerConfiguration.getTransactionsEnabled()) {
+ propertiesProducer.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG,
+ new TransactionIdSupplier(producerConfiguration.getTransactionIdPrefix()).get());
+ }
+ if (producerConfiguration.getDeliveryGuarantee() != null) {
+ propertiesProducer.put(ProducerConfig.ACKS_CONFIG, producerConfiguration.getDeliveryGuarantee());
+ }
+ if (producerConfiguration.getCompressionCodec() != null) {
+ propertiesProducer.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerConfiguration.getCompressionCodec());
+ }
+ final String partitionClass = producerConfiguration.getPartitionClass();
+ if (partitionClass != null && partitionClass.startsWith("org.apache.kafka")) {
+ propertiesProducer.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, partitionClass);
+ }
+ return new Kafka3ProducerService(propertiesProducer, serviceConfiguration, producerConfiguration);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verify(final ConfigurationContext configurationContext, final ComponentLog verificationLogger, final Map<String, String> variables) {
+ final List<ConfigVerificationResult> results = new ArrayList<>();
+
+ final Properties clientProperties = getClientProperties(configurationContext);
+ clientProperties.putAll(variables);
+ try (final Admin admin = Admin.create(clientProperties)) {
+ final ListTopicsResult listTopicsResult = admin.listTopics();
+
+ final KafkaFuture<Collection<TopicListing>> requestedListings = listTopicsResult.listings();
+ final Collection<TopicListing> topicListings = requestedListings.get(VERIFY_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
+ final String topicListingExplanation = String.format("Topics Found [%d]", topicListings.size());
+ results.add(
+ new ConfigVerificationResult.Builder()
+ .verificationStepName(TOPIC_LISTING_STEP)
+ .outcome(SUCCESSFUL)
+ .explanation(topicListingExplanation)
+ .build()
+ );
+ } catch (final Exception e) {
+ verificationLogger.error("Kafka Broker verification failed", e);
+ results.add(
+ new ConfigVerificationResult.Builder()
+ .verificationStepName(CONNECTION_STEP)
+ .outcome(FAILED)
+ .explanation(e.getMessage())
+ .build()
+ );
+ }
+
+ return results;
+ }
+
+ private Properties getConsumerProperties(final PropertyContext propertyContext, final Properties defaultProperties) {
+ final Properties properties = new Properties();
+ properties.putAll(defaultProperties);
+
+ final IsolationLevel isolationLevel = propertyContext.getProperty(TRANSACTION_ISOLATION_LEVEL).asAllowableValue(IsolationLevel.class);
+ properties.put(TRANSACTION_ISOLATION_LEVEL.getName(), isolationLevel.getValue());
+
+ return properties;
+ }
+
+ private Properties getClientProperties(final PropertyContext propertyContext) {
+ final Properties properties = new Properties();
+
+ final KafkaPropertyProvider propertyProvider = new StandardKafkaPropertyProvider(ConsumerConfig.class);
+ final Map<String, Object> propertiesProvider = propertyProvider.getProperties(propertyContext);
+ propertiesProvider.forEach((key, value) -> properties.setProperty(key, value.toString()));
+
+ final String configuredBootstrapServers = propertyContext.getProperty(BOOTSTRAP_SERVERS).getValue();
+ properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, configuredBootstrapServers);
+
+ setSslProperties(properties, propertyContext);
+
+ final int defaultApiTimeoutMs = getDefaultApiTimeoutMs(propertyContext);
+ properties.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, defaultApiTimeoutMs);
+
+ final int requestTimeoutMs = getRequestTimeoutMs(propertyContext);
+ properties.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs);
+
+ final long timePeriod = propertyContext.getProperty(METADATA_WAIT_TIME).asTimePeriod(TimeUnit.MILLISECONDS);
+ properties.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, timePeriod);
+
+ return properties;
+ }
+
+ private void setSslProperties(final Properties properties, final PropertyContext context) {
+ final PropertyValue sslContextServiceProperty = context.getProperty(SSL_CONTEXT_SERVICE);
+ if (sslContextServiceProperty.isSet()) {
+ final SSLContextService sslContextService = sslContextServiceProperty.asControllerService(SSLContextService.class);
+ if (sslContextService.isKeyStoreConfigured()) {
+ properties.put(SSL_KEYSTORE_LOCATION.getProperty(), sslContextService.getKeyStoreFile());
+ properties.put(SSL_KEYSTORE_TYPE.getProperty(), sslContextService.getKeyStoreType());
+
+ final String keyStorePassword = sslContextService.getKeyStorePassword();
+ properties.put(SSL_KEYSTORE_PASSWORD.getProperty(), keyStorePassword);
+
+ final String keyPassword = sslContextService.getKeyPassword();
+ final String configuredKeyPassword = keyPassword == null ? keyStorePassword : keyPassword;
+ properties.put(SSL_KEY_PASSWORD.getProperty(), configuredKeyPassword);
+ }
+ if (sslContextService.isTrustStoreConfigured()) {
+ properties.put(SSL_TRUSTSTORE_LOCATION.getProperty(), sslContextService.getTrustStoreFile());
+ properties.put(SSL_TRUSTSTORE_TYPE.getProperty(), sslContextService.getTrustStoreType());
+ properties.put(SSL_TRUSTSTORE_PASSWORD.getProperty(), sslContextService.getTrustStorePassword());
+ }
+ }
+ }
+
+ private ServiceConfiguration getServiceConfiguration(final PropertyContext propertyContext) {
+ final Duration maxAckWait = propertyContext.getProperty(ACK_WAIT_TIME).asDuration();
+ return new ServiceConfiguration(maxAckWait);
+ }
+
+ private int getDefaultApiTimeoutMs(final PropertyContext propertyContext) {
+ return propertyContext.getProperty(CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
+ }
+
+ private int getRequestTimeoutMs(final PropertyContext propertyContext) {
+ final int defaultApiTimeoutMs = getDefaultApiTimeoutMs(propertyContext);
+ return defaultApiTimeoutMs / 2;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/ConsumerException.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/ConsumerException.java
index 15aa16ed76..c4f0977ba7 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/ConsumerException.java
@@ -14,31 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.kafka.service.consumer;
/**
- * Kafka FlowFile Attributes
+ * Kafka Consumer Exception indicating problems interacting with consumer objects
*/
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
+public class ConsumerException extends RuntimeException {
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
-
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
-
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
-
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
-
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+ public ConsumerException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
new file mode 100644
index 0000000000..06864452a3
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.consumer;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.nifi.kafka.service.api.common.OffsetSummary;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.common.TopicPartitionSummary;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
+import org.apache.nifi.kafka.service.api.consumer.PollingContext;
+import org.apache.nifi.kafka.service.api.consumer.PollingSummary;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.service.consumer.pool.ConsumerObjectPool;
+import org.apache.nifi.kafka.service.consumer.pool.Subscription;
+import org.apache.nifi.logging.ComponentLog;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Kafka 3 Consumer Service implementation with Object Pooling for subscribed Kafka Consumers
+ */
+public class Kafka3ConsumerService implements KafkaConsumerService {
+ private final ComponentLog componentLog;
+
+ private final ConsumerObjectPool consumerObjectPool;
+
+ public Kafka3ConsumerService(final ComponentLog componentLog, final Properties properties) {
+ this.componentLog = Objects.requireNonNull(componentLog, "Component Log required");
+ this.consumerObjectPool = new ConsumerObjectPool(properties);
+ }
+
+ @Override
+ public void commit(final PollingSummary pollingSummary) {
+ Objects.requireNonNull(pollingSummary, "Polling Summary required");
+
+ final Subscription subscription = getSubscription(pollingSummary);
+ final Map<TopicPartition, OffsetAndMetadata> offsets = getOffsets(pollingSummary);
+
+ final long started = System.currentTimeMillis();
+ final long elapsed = runConsumerFunction(subscription, (consumer) -> {
+ consumer.commitSync(offsets);
+ return started - System.currentTimeMillis();
+ });
+
+ componentLog.debug("Committed Records in [{} ms] for {}", elapsed, pollingSummary);
+ }
+
+ @Override
+ public Iterable<ByteRecord> poll(final PollingContext pollingContext) {
+ Objects.requireNonNull(pollingContext, "Polling Context required");
+ final Subscription subscription = getSubscription(pollingContext);
+
+ return runConsumerFunction(subscription, (consumer) -> {
+ final ConsumerRecords<byte[], byte[]> consumerRecords = consumer.poll(pollingContext.getMaxUncommittedTime());
+ return new RecordIterable(consumerRecords);
+ });
+ }
+
+ @Override
+ public List<PartitionState> getPartitionStates(final PollingContext pollingContext) {
+ final Subscription subscription = getSubscription(pollingContext);
+ final Iterator<String> topics = subscription.getTopics().iterator();
+
+ final List<PartitionState> partitionStates;
+
+ if (topics.hasNext()) {
+ final String topic = topics.next();
+ partitionStates = runConsumerFunction(subscription, (consumer) ->
+ consumer.partitionsFor(topic)
+ .stream()
+ .map(partitionInfo -> new PartitionState(partitionInfo.topic(), partitionInfo.partition()))
+ .collect(Collectors.toList())
+ );
+ } else {
+ partitionStates = Collections.emptyList();
+ }
+
+ return partitionStates;
+ }
+
+ @Override
+ public void close() {
+ consumerObjectPool.close();
+ }
+
+ private Subscription getSubscription(final PollingContext pollingContext) {
+ final String groupId = pollingContext.getGroupId();
+ final Optional<Pattern> topicPatternFound = pollingContext.getTopicPattern();
+ final AutoOffsetReset autoOffsetReset = pollingContext.getAutoOffsetReset();
+
+ return topicPatternFound
+ .map(pattern -> new Subscription(groupId, pattern, autoOffsetReset))
+ .orElseGet(() -> new Subscription(groupId, pollingContext.getTopics(), autoOffsetReset));
+ }
+
+ private Map<TopicPartition, OffsetAndMetadata> getOffsets(final PollingSummary pollingSummary) {
+ final Map<TopicPartition, OffsetAndMetadata> offsets = new LinkedHashMap<>();
+
+ final Map<TopicPartitionSummary, OffsetSummary> summaryOffsets = pollingSummary.getOffsets();
+ for (final Map.Entry<TopicPartitionSummary, OffsetSummary> offsetEntry : summaryOffsets.entrySet()) {
+ final TopicPartitionSummary topicPartitionSummary = offsetEntry.getKey();
+ final TopicPartition topicPartition = new TopicPartition(topicPartitionSummary.getTopic(), topicPartitionSummary.getPartition());
+
+ final OffsetSummary offsetSummary = offsetEntry.getValue();
+ final OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(offsetSummary.getOffset());
+ offsets.put(topicPartition, offsetAndMetadata);
+ }
+
+ return offsets;
+ }
+
+ private <T> T runConsumerFunction(final Subscription subscription, final Function<Consumer<byte[], byte[]>, T> consumerFunction) {
+ Consumer<byte[], byte[]> consumer = null;
+ try {
+ consumer = consumerObjectPool.borrowObject(subscription);
+ return consumerFunction.apply(consumer);
+ } catch (final Exception e) {
+ throw new ConsumerException("Borrow Consumer failed", e);
+ } finally {
+ if (consumer != null) {
+ try {
+ consumerObjectPool.returnObject(subscription, consumer);
+ } catch (final Exception e) {
+ componentLog.warn("Return Consumer failed", e);
+ }
+ }
+ }
+ }
+
+ private static class RecordIterable implements Iterable<ByteRecord> {
+ private final Iterator<ByteRecord> records;
+
+ private RecordIterable(final Iterable<ConsumerRecord<byte[], byte[]>> consumerRecords) {
+ this.records = new RecordIterator(consumerRecords);
+ }
+
+ @Override
+ public Iterator<ByteRecord> iterator() {
+ return records;
+ }
+ }
+
+ private static class RecordIterator implements Iterator<ByteRecord> {
+ private final Iterator<ConsumerRecord<byte[], byte[]>> consumerRecords;
+
+ private RecordIterator(final Iterable<ConsumerRecord<byte[], byte[]>> records) {
+ this.consumerRecords = records.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return consumerRecords.hasNext();
+ }
+
+ @Override
+ public ByteRecord next() {
+ final ConsumerRecord<byte[], byte[]> consumerRecord = consumerRecords.next();
+ final List<RecordHeader> recordHeaders = new ArrayList<>();
+ consumerRecord.headers().forEach(header -> {
+ final RecordHeader recordHeader = new RecordHeader(header.key(), header.value());
+ recordHeaders.add(recordHeader);
+ });
+ return new ByteRecord(
+ consumerRecord.topic(),
+ consumerRecord.partition(),
+ consumerRecord.offset(),
+ consumerRecord.timestamp(),
+ recordHeaders,
+ consumerRecord.key(),
+ consumerRecord.value()
+ );
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerFactory.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerFactory.java
index 15aa16ed76..45c9b37851 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerFactory.java
@@ -14,31 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.kafka.service.consumer.pool;
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
+import org.apache.kafka.clients.consumer.Consumer;
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
+import java.util.Properties;
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
-
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
-
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+/**
+ * Factory abstraction for creating Kafka Consumer objects
+ */
+interface ConsumerFactory {
+ /**
+ * Create new Kafka Consumer using supplied Properties
+ *
+ * @param properties Consumer configuration properties
+ * @return Kafka Consumer
+ */
+ Consumer<byte[], byte[]> newConsumer(Properties properties);
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerObjectPool.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerObjectPool.java
index 15aa16ed76..fbac887ceb 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerObjectPool.java
@@ -14,31 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.kafka.service.consumer.pool;
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
+import java.util.Properties;
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
-
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
-
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+/**
+ * Kafka Consumer Object Pool
+ */
+public class ConsumerObjectPool extends GenericKeyedObjectPool<Subscription, Consumer<byte[], byte[]>> {
+ /**
+ * Consumer Object Pool constructor with Kafka Consumer Properties
+ *
+ * @param consumerProperties Kafka Consumer Properties required
+ */
+ public ConsumerObjectPool(final Properties consumerProperties) {
+ super(new ConsumerPooledObjectFactory(consumerProperties, new StandardConsumerFactory()));
+ }
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerPooledObjectFactory.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerPooledObjectFactory.java
new file mode 100644
index 0000000000..6b91340bbd
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerPooledObjectFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.consumer.pool;
+
+import org.apache.commons.pool2.BaseKeyedPooledObjectFactory;
+import org.apache.commons.pool2.PooledObject;
+import org.apache.commons.pool2.impl.DefaultPooledObject;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.regex.Pattern;
+
+/**
+ * Pooled Object Factory for Kafka Consumers
+ */
+class ConsumerPooledObjectFactory extends BaseKeyedPooledObjectFactory<Subscription, Consumer<byte[], byte[]>> {
+ private final Properties consumerProperties;
+
+ private final ConsumerFactory consumerFactory;
+
+ /**
+ * Consumer Pooled Object Factory constructor with Kafka Consumer Properties
+ *
+ * @param consumerProperties Kafka Consumer Properties
+ * @param consumerFactory Kafka Consumer Factory
+ */
+ ConsumerPooledObjectFactory(final Properties consumerProperties, final ConsumerFactory consumerFactory) {
+ this.consumerProperties = Objects.requireNonNull(consumerProperties, "Consumer Properties required");
+ this.consumerFactory = Objects.requireNonNull(consumerFactory, "Consumer Factory required");
+ }
+
+ @Override
+ public Consumer<byte[], byte[]> create(final Subscription subscription) {
+ Objects.requireNonNull(subscription, "Topic Subscription required");
+
+ final Properties properties = new Properties();
+ properties.putAll(consumerProperties);
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, subscription.getGroupId());
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, subscription.getAutoOffsetReset().getValue());
+
+ final Consumer<byte[], byte[]> consumer = consumerFactory.newConsumer(properties);
+
+ final Optional<Pattern> topicPatternFound = subscription.getTopicPattern();
+ if (topicPatternFound.isPresent()) {
+ final Pattern topicPattern = topicPatternFound.get();
+ consumer.subscribe(topicPattern);
+ } else {
+ final Collection<String> topics = subscription.getTopics();
+ consumer.subscribe(topics);
+ }
+
+ return consumer;
+ }
+
+ /**
+ * Wrap Kafka Consumer using Default Pooled Object for tracking
+ *
+ * @param consumer Kafka Consumer
+ * @return Pooled Object wrapper
+ */
+ @Override
+ public PooledObject<Consumer<byte[], byte[]>> wrap(final Consumer<byte[], byte[]> consumer) {
+ Objects.requireNonNull(consumer, "Consumer required");
+ return new DefaultPooledObject<>(consumer);
+ }
+
+ /**
+ * Destroy Pooled Object closes wrapped Kafka Consumer
+ *
+ * @param subscription Subscription
+ * @param pooledObject Pooled Object with Consumer to be closed
+ */
+ @Override
+ public void destroyObject(final Subscription subscription, final PooledObject<Consumer<byte[], byte[]>> pooledObject) {
+ Objects.requireNonNull(pooledObject, "Pooled Object required");
+ final Consumer<byte[], byte[]> consumer = pooledObject.getObject();
+ consumer.unsubscribe();
+ consumer.close();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/StandardConsumerFactory.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/StandardConsumerFactory.java
new file mode 100644
index 0000000000..8cbec41007
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/StandardConsumerFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.consumer.pool;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+
+import java.util.Objects;
+import java.util.Properties;
+
+/**
+ * Standard Kafka Consumer Factory with Byte Array Deserializer for Key and Value elements
+ */
+class StandardConsumerFactory implements ConsumerFactory {
+ /**
+ * Create new Kafka Consumer with Byte Array Deserializer and configured properties
+ *
+ * @param properties Consumer configuration properties
+ * @return Kafka Consumer
+ */
+ @Override
+ public Consumer<byte[], byte[]> newConsumer(final Properties properties) {
+ Objects.requireNonNull(properties, "Properties required");
+ final ByteArrayDeserializer deserializer = new ByteArrayDeserializer();
+ return new KafkaConsumer<>(properties, deserializer, deserializer);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/Subscription.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/Subscription.java
new file mode 100644
index 0000000000..7f5a08eb45
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/consumer/pool/Subscription.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.consumer.pool;
+
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+/**
+ * Subscription for pooled Kafka Consumers
+ */
+public class Subscription {
+ private final String groupId;
+
+ private final Collection<String> topics;
+
+ private final Pattern topicPattern;
+
+ private final AutoOffsetReset autoOffsetReset;
+
+ public Subscription(final String groupId, final Collection<String> topics, final AutoOffsetReset autoOffsetReset) {
+ this.groupId = Objects.requireNonNull(groupId, "Group ID required");
+ this.topics = Collections.unmodifiableCollection(Objects.requireNonNull(topics, "Topics required"));
+ this.topicPattern = null;
+ this.autoOffsetReset = Objects.requireNonNull(autoOffsetReset, "Auto Offset Reset required");
+ }
+
+ public Subscription(final String groupId, final Pattern topicPattern, final AutoOffsetReset autoOffsetReset) {
+ this.groupId = Objects.requireNonNull(groupId, "Group ID required");
+ this.topics = Collections.emptyList();
+ this.topicPattern = Objects.requireNonNull(topicPattern, "Topic Pattern required");
+ this.autoOffsetReset = Objects.requireNonNull(autoOffsetReset, "Auto Offset Reset required");
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public Collection<String> getTopics() {
+ return topics;
+ }
+
+ public Optional<Pattern> getTopicPattern() {
+ return Optional.ofNullable(topicPattern);
+ }
+
+ public AutoOffsetReset getAutoOffsetReset() {
+ return autoOffsetReset;
+ }
+
+ @Override
+ public boolean equals(final Object object) {
+ final boolean equals;
+
+ if (object == null) {
+ equals = false;
+ } else if (object instanceof Subscription) {
+ final Subscription subscription = (Subscription) object;
+ if (groupId.equals(subscription.groupId)) {
+ if (isTopicSubscriptionMatched(subscription)) {
+ equals = autoOffsetReset == subscription.autoOffsetReset;
+ } else {
+ equals = false;
+ }
+ } else {
+ equals = false;
+ }
+ } else {
+ equals = false;
+ }
+
+ return equals;
+ }
+
+ @Override
+ public int hashCode() {
+ return groupId.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Subscription Group ID [%s] Topics %s Topic Pattern [%s]", groupId, topics, topicPattern);
+ }
+
+ private boolean isTopicSubscriptionMatched(final Subscription subscription) {
+ final boolean matched;
+
+ if (topics.size() == subscription.topics.size()) {
+ if (topics.containsAll(subscription.topics)) {
+ final String regexLeft = (topicPattern == null ? null : topicPattern.pattern());
+ final String regexRight = (subscription.topicPattern == null ? null : subscription.topicPattern.pattern());
+ matched = Objects.equals(regexLeft, regexRight);
+ } else {
+ matched = false;
+ }
+ } else {
+ matched = false;
+ }
+
+ return matched;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/Kafka3ProducerService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/Kafka3ProducerService.java
new file mode 100644
index 0000000000..6f97e94f5c
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/Kafka3ProducerService.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.producer;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.common.ServiceConfiguration;
+import org.apache.nifi.kafka.service.api.producer.FlowFileResult;
+import org.apache.nifi.kafka.service.api.producer.KafkaProducerService;
+import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration;
+import org.apache.nifi.kafka.service.api.producer.PublishContext;
+import org.apache.nifi.kafka.service.api.producer.RecordSummary;
+import org.apache.nifi.kafka.service.api.record.KafkaRecord;
+import org.apache.nifi.kafka.service.producer.transaction.KafkaNonTransactionalProducerWrapper;
+import org.apache.nifi.kafka.service.producer.transaction.KafkaProducerWrapper;
+import org.apache.nifi.kafka.service.producer.transaction.KafkaTransactionalProducerWrapper;
+
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+public class Kafka3ProducerService implements KafkaProducerService {
+ private final Producer<byte[], byte[]> producer;
+
+ private final List<ProducerCallback> callbacks;
+
+ private final ServiceConfiguration serviceConfiguration;
+
+ private final KafkaProducerWrapper wrapper;
+
+ public Kafka3ProducerService(final Properties properties,
+ final ServiceConfiguration serviceConfiguration,
+ final ProducerConfiguration producerConfiguration) {
+ final ByteArraySerializer serializer = new ByteArraySerializer();
+ this.producer = new KafkaProducer<>(properties, serializer, serializer);
+ this.callbacks = new ArrayList<>();
+
+ this.serviceConfiguration = serviceConfiguration;
+
+ this.wrapper = producerConfiguration.getTransactionsEnabled()
+ ? new KafkaTransactionalProducerWrapper(producer)
+ : new KafkaNonTransactionalProducerWrapper(producer);
+ }
+
+ @Override
+ public void close() {
+ producer.close();
+ }
+
+ @Override
+ public void init() {
+ wrapper.init();
+ }
+
+ @Override
+ public void send(final Iterator<KafkaRecord> kafkaRecords, final PublishContext publishContext) {
+ final ProducerCallback callback = new ProducerCallback(publishContext.getFlowFile());
+ callbacks.add(callback);
+ Optional.ofNullable(publishContext.getException()).ifPresent(e -> callback.getExceptions().add(e));
+ if (callback.getExceptions().isEmpty()) {
+ try {
+ wrapper.send(kafkaRecords, publishContext, callback);
+ } catch (final UncheckedIOException e) {
+ callback.getExceptions().add(e);
+ }
+ }
+ }
+
+ @Override
+ public RecordSummary complete() {
+ final boolean shouldCommit = callbacks.stream().noneMatch(ProducerCallback::isFailure);
+ if (shouldCommit) {
+ producer.flush(); // finish Kafka processing of in-flight data
+ wrapper.commit(); // commit Kafka transaction (when transactions configured)
+ } else {
+ // rollback on transactions + exception
+ wrapper.abort();
+ }
+
+ final RecordSummary recordSummary = new RecordSummary(); // scrape the Kafka callbacks for disposition of in-flight data
+ final List<FlowFileResult> flowFileResults = recordSummary.getFlowFileResults();
+ for (final ProducerCallback callback : callbacks) {
+ // short-circuit the handling of the flowfile results here
+ if (callback.isFailure()) {
+ flowFileResults.add(callback.toFailureResult());
+ } else {
+ flowFileResults.add(callback.waitComplete(serviceConfiguration.getMaxAckWait().toMillis()));
+ }
+ }
+ return recordSummary;
+ }
+
+ @Override
+ public List<PartitionState> getPartitionStates(final String topic) {
+ final List<PartitionInfo> partitionInfos = producer.partitionsFor(topic);
+ return partitionInfos.stream()
+ .map(p -> new PartitionState(p.topic(), p.partition()))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/ProducerCallback.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/ProducerCallback.java
new file mode 100644
index 0000000000..92dafe43af
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/ProducerCallback.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.producer;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.kafka.service.api.producer.FlowFileResult;
+import org.apache.nifi.kafka.service.api.producer.ProducerRecordMetadata;
+import org.apache.nifi.kafka.shared.util.Notifier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+public class ProducerCallback implements Callback {
+ private final AtomicLong sentCount;
+ private final AtomicLong acknowledgedCount;
+ private final AtomicLong failedCount;
+ private final FlowFile flowFile;
+ private final List<ProducerRecordMetadata> metadatas;
+ private final List<Exception> exceptions;
+ private final Notifier notifier;
+
+ public List<Exception> getExceptions() {
+ return exceptions;
+ }
+
+ public boolean isFailure() {
+ return !exceptions.isEmpty();
+ }
+
+ public ProducerCallback(final FlowFile flowFile) {
+ this.sentCount = new AtomicLong(0L);
+ this.acknowledgedCount = new AtomicLong(0L);
+ this.failedCount = new AtomicLong(0L);
+ this.flowFile = flowFile;
+ this.metadatas = new ArrayList<>();
+ this.exceptions = new ArrayList<>();
+ this.notifier = new Notifier();
+ }
+
+ public long send() {
+ return sentCount.incrementAndGet();
+ }
+
+ @Override
+ public void onCompletion(final RecordMetadata metadata, final Exception exception) {
+ if (exception == null) {
+ acknowledgedCount.addAndGet(1L);
+ metadatas.add(toProducerRecordMetadata(metadata));
+ } else {
+ failedCount.addAndGet(1L);
+ exceptions.add(exception);
+ }
+ notifier.notifyWaiter();
+ }
+
+ private static ProducerRecordMetadata toProducerRecordMetadata(final RecordMetadata m) {
+ return new ProducerRecordMetadata(m.topic(), m.partition(), m.offset(), m.timestamp());
+ }
+
+ public FlowFileResult waitComplete(final long maxAckWaitMillis) {
+ final Supplier<Boolean> conditionComplete = () -> ((acknowledgedCount.get() + failedCount.get()) == sentCount.get());
+ notifier.waitForCondition(conditionComplete, maxAckWaitMillis);
+ return new FlowFileResult(flowFile, sentCount.get(), metadatas, exceptions);
+ }
+
+ public FlowFileResult toFailureResult() {
+ return new FlowFileResult(flowFile, sentCount.get(), metadatas, exceptions);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaNonTransactionalProducerWrapper.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaNonTransactionalProducerWrapper.java
index 15aa16ed76..feee4218ad 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaNonTransactionalProducerWrapper.java
@@ -14,31 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.kafka.service.producer.transaction;
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
-
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
+import org.apache.kafka.clients.producer.Producer;
- String KAFKA_TIMESTAMP = "kafka.timestamp";
+public class KafkaNonTransactionalProducerWrapper extends KafkaProducerWrapper {
- String KAFKA_COUNT = "kafka.count";
+ public KafkaNonTransactionalProducerWrapper(final Producer<byte[], byte[]> producer) {
+ super(producer);
+ }
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
+ @Override
+ public void init() {
+ }
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
+ @Override
+ public void commit() {
+ }
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+ @Override
+ public void abort() {
+ }
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaProducerWrapper.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaProducerWrapper.java
new file mode 100644
index 0000000000..39f71b4ae7
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaProducerWrapper.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.producer.transaction;
+
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.nifi.kafka.service.api.producer.PublishContext;
+import org.apache.nifi.kafka.service.api.record.KafkaRecord;
+import org.apache.nifi.kafka.service.producer.ProducerCallback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Abstract away the configured transactionality of the PublishKafka producer.
+ */
+public abstract class KafkaProducerWrapper {
+ protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+ protected final Producer<byte[], byte[]> producer;
+
+ public KafkaProducerWrapper(final Producer<byte[], byte[]> producer) {
+ this.producer = producer;
+ }
+
+ /**
+ * Transaction-enabled publish to Kafka involves the use of special Kafka client library APIs.
+ */
+ public abstract void init();
+
+ public void send(final Iterator<KafkaRecord> kafkaRecords, final PublishContext publishContext, final ProducerCallback callback) {
+ while (kafkaRecords.hasNext()) {
+ final KafkaRecord kafkaRecord = kafkaRecords.next();
+ producer.send(toProducerRecord(kafkaRecord, publishContext), callback);
+ callback.send();
+ }
+ }
+
+ /**
+ * Transaction-enabled publish to Kafka involves the use of special Kafka client library APIs.
+ */
+ public abstract void commit();
+
+ /**
+ * Transaction-enabled publish to Kafka involves the use of special Kafka client library APIs.
+ */
+ public abstract void abort();
+
+ private ProducerRecord<byte[], byte[]> toProducerRecord(final KafkaRecord kafkaRecord, final PublishContext publishContext) {
+ final String topic = Optional.ofNullable(kafkaRecord.getTopic()).orElse(publishContext.getTopic());
+ final Integer partition = Optional.ofNullable(kafkaRecord.getPartition()).orElse(publishContext.getPartition());
+ final Integer moddedPartition = partition == null ? null : Math.abs(partition) % (producer.partitionsFor(topic).size());
+ return new ProducerRecord<>(topic, moddedPartition, kafkaRecord.getTimestamp(), kafkaRecord.getKey(), kafkaRecord.getValue(), toKafkaHeadersNative(kafkaRecord));
+ }
+
+ private List<Header> toKafkaHeadersNative(final KafkaRecord kafkaRecord) {
+ return kafkaRecord.getHeaders().stream()
+ .map(h -> new RecordHeader(h.key(), h.value()))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaTransactionalProducerWrapper.java
similarity index 51%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaTransactionalProducerWrapper.java
index 15aa16ed76..ab027aabaf 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/java/org/apache/nifi/kafka/service/producer/transaction/KafkaTransactionalProducerWrapper.java
@@ -14,31 +14,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
-
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
-
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
-
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
-
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
-
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
-
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+package org.apache.nifi.kafka.service.producer.transaction;
+
+import org.apache.kafka.clients.producer.Producer;
+
+public class KafkaTransactionalProducerWrapper extends KafkaProducerWrapper {
+
+ public KafkaTransactionalProducerWrapper(final Producer<byte[], byte[]> producer) {
+ super(producer);
+ }
+
+ @Override
+ public void init() {
+ producer.initTransactions();
+ producer.beginTransaction();
+ }
+
+ @Override
+ public void commit() {
+ try {
+ producer.commitTransaction();
+ } catch (final Exception e) {
+ logger.debug("Failure during producer transaction commit", e);
+ abort();
+ }
+ }
+
+ @Override
+ public void abort() {
+ producer.abortTransaction();
+ }
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
new file mode 100644
index 0000000000..2c023932a5
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.kafka.service.Kafka3ConnectionService
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceBaseIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceBaseIT.java
new file mode 100644
index 0000000000..5435e63b77
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceBaseIT.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service;
+
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.CreateTopicsResult;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.security.plain.PlainLoginModule;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
+import org.apache.nifi.kafka.service.api.consumer.PollingContext;
+import org.apache.nifi.kafka.service.api.producer.KafkaProducerService;
+import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration;
+import org.apache.nifi.kafka.service.api.producer.PublishContext;
+import org.apache.nifi.kafka.service.api.producer.RecordSummary;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.service.api.record.KafkaRecord;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.security.util.TemporaryKeyStoreBuilder;
+import org.apache.nifi.security.util.TlsConfiguration;
+import org.apache.nifi.ssl.SSLContextService;
+import org.apache.nifi.ssl.StandardRestrictedSSLContextService;
+import org.apache.nifi.util.MockConfigurationContext;
+import org.apache.nifi.util.NoOpProcessor;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestInstance.Lifecycle;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@TestInstance(Lifecycle.PER_CLASS)
+public class Kafka3ConnectionServiceBaseIT {
+ // This Base class executes its tests with Ssl off and Sasl off.
+ // There are subclasses which execute these same tests and enable Ssl or Sasl
+
+ public static final String IMAGE_NAME = "confluentinc/cp-kafka:7.6.1"; // April 2024
+
+ private static final String GROUP_ID = Kafka3ConnectionService.class.getSimpleName();
+
+ private static final String TOPIC = Kafka3ConnectionServiceBaseIT.class.getSimpleName();
+
+ private static final String SERVICE_ID = Kafka3ConnectionService.class.getSimpleName();
+
+ private static final String UNREACHABLE_BOOTSTRAP_SERVERS = "127.0.0.1:1000";
+
+ private static final String CLIENT_TIMEOUT = "2 s";
+
+ private static final String TEST_RECORD_VALUE = "value-" + System.currentTimeMillis();
+
+ private static final byte[] RECORD_KEY = new byte[]{1};
+
+ private static final byte[] RECORD_VALUE = TEST_RECORD_VALUE.getBytes(StandardCharsets.UTF_8);
+
+ private static final int POLLING_ATTEMPTS = 3;
+
+ private static final Set<String> fileLocationNames = Set.of(
+ "KAFKA_SSL_KEYSTORE_LOCATION", "KAFKA_SSL_TRUSTSTORE_LOCATION");
+
+ protected static final String TEST_USERNAME = "nifi";
+ protected static final String TEST_PASSWORD = UUID.randomUUID().toString();
+
+ protected TlsConfiguration tlsConfiguration;
+
+ protected TestRunner runner;
+
+ private KafkaContainer kafkaContainer;
+
+ private Kafka3ConnectionService service;
+
+ @BeforeAll
+ void startContainer() {
+ tlsConfiguration = new TemporaryKeyStoreBuilder().build();
+ kafkaContainer = new KafkaContainer(DockerImageName.parse(IMAGE_NAME));
+ initializeContainer();
+ kafkaContainer.start();
+ }
+
+ @BeforeEach
+ void setRunner() throws InitializationException {
+ runner = TestRunners.newTestRunner(NoOpProcessor.class);
+ service = new Kafka3ConnectionService();
+ runner.addControllerService(SERVICE_ID, service, getKafkaServiceConfigProperties());
+ runner.enableControllerService(service);
+ }
+
+ @AfterEach
+ void cleanup() {
+ runner.disableControllerService(service);
+ }
+
+ @AfterAll
+ void stopContainer() {
+ kafkaContainer.stop();
+ }
+
+ protected void initializeContainer() {
+ Map<String, String> environment = getKafkaContainerConfigProperties();
+ kafkaContainer.withEnv(environment);
+
+ // For each property which is a file location, copy the file into the kafka container.
+ environment.entrySet().stream()
+ .filter(e -> fileLocationNames.contains(e.getKey()))
+ .forEach(e -> kafkaContainer.withCopyFileToContainer(
+ MountableFile.forHostPath(e.getValue()), e.getValue()));
+ }
+
+ protected Map<String, String> getKafkaContainerConfigProperties() {
+ return Collections.emptyMap();
+ }
+
+ protected Map<String, String> getKafkaServiceConfigProperties() throws InitializationException {
+ final Map<String, String> properties = new LinkedHashMap<>();
+ properties.put(Kafka3ConnectionService.BOOTSTRAP_SERVERS.getName(), kafkaContainer.getBootstrapServers());
+ properties.put(Kafka3ConnectionService.CLIENT_TIMEOUT.getName(), CLIENT_TIMEOUT);
+ return properties;
+ }
+
+ protected Map<String, String> getAdminClientConfigProperties() {
+ return Collections.emptyMap();
+ }
+
+ @Test
+ void testAdminClient() throws ExecutionException, InterruptedException, TimeoutException {
+ final Properties properties = new Properties();
+ properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());
+ properties.putAll(getAdminClientConfigProperties());
+
+ try (final Admin adminClient = Admin.create(properties)) {
+ final int numPartitions = 1;
+ final short replicationFactor = 1;
+ final NewTopic newTopic = new NewTopic(TOPIC, numPartitions, replicationFactor);
+ final CreateTopicsResult topics = adminClient.createTopics(Collections.singleton(newTopic));
+ final KafkaFuture<Void> topicFuture = topics.values().get(TOPIC);
+ topicFuture.get(2, TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ void testProduceOneNoTransaction() {
+ final ProducerConfiguration producerConfiguration = new ProducerConfiguration(false, null, null, null, null);
+ final KafkaProducerService producerService = service.getProducerService(producerConfiguration);
+ final KafkaRecord kafkaRecord = new KafkaRecord(null, null, null, null, RECORD_VALUE, Collections.emptyList());
+ final List<KafkaRecord> kafkaRecords = Collections.singletonList(kafkaRecord);
+ producerService.init();
+ producerService.send(kafkaRecords.iterator(), new PublishContext(TOPIC + "-produce", null, null, null));
+ final RecordSummary summary = producerService.complete();
+ assertNotNull(summary);
+ }
+
+ @Test
+ void testProduceOneWithTransaction() {
+ final ProducerConfiguration producerConfiguration = new ProducerConfiguration(true, "transaction-", null, null, null);
+ final KafkaProducerService producerService = service.getProducerService(producerConfiguration);
+ final KafkaRecord kafkaRecord = new KafkaRecord(null, null, null, null, RECORD_VALUE, Collections.emptyList());
+ final List<KafkaRecord> kafkaRecords = Collections.singletonList(kafkaRecord);
+ producerService.init();
+ producerService.send(kafkaRecords.iterator(), new PublishContext(TOPIC + "-produce", null, null, null));
+ final RecordSummary summary = producerService.complete();
+ assertNotNull(summary);
+ }
+
+ @Test
+ void testProduceConsumeRecord() throws Exception {
+ final ProducerConfiguration producerConfiguration = new ProducerConfiguration(false, null, null, null, null);
+ final KafkaProducerService producerService = service.getProducerService(producerConfiguration);
+
+ final long timestamp = System.currentTimeMillis();
+ final KafkaRecord kafkaRecord = new KafkaRecord(null, null, timestamp, RECORD_KEY, RECORD_VALUE, Collections.emptyList());
+ final List<KafkaRecord> kafkaRecords = Collections.singletonList(kafkaRecord);
+ producerService.init();
+ producerService.send(kafkaRecords.iterator(), new PublishContext(TOPIC, null, null, null));
+ final RecordSummary summary = producerService.complete();
+ assertNotNull(summary);
+
+ try (KafkaConsumerService consumerService = service.getConsumerService(null)) {
+ final PollingContext pollingContext = new PollingContext(
+ GROUP_ID, Collections.singleton(TOPIC), AutoOffsetReset.EARLIEST, Duration.ofSeconds(1));
+ final Iterator<ByteRecord> consumerRecords = poll(consumerService, pollingContext);
+
+ assertTrue(consumerRecords.hasNext(), "Consumer Records not found");
+
+ final ByteRecord consumerRecord = consumerRecords.next();
+ assertEquals(TOPIC, consumerRecord.getTopic());
+ assertEquals(0, consumerRecord.getOffset());
+ assertEquals(0, consumerRecord.getPartition());
+ assertEquals(timestamp, consumerRecord.getTimestamp());
+
+ final Optional<byte[]> keyFound = consumerRecord.getKey();
+ assertTrue(keyFound.isPresent());
+
+ assertArrayEquals(RECORD_KEY, keyFound.get());
+ assertArrayEquals(RECORD_VALUE, consumerRecord.getValue());
+
+ assertFalse(consumerRecords.hasNext());
+ }
+ }
+
+ @Test
+ void testVerifySuccessful() throws InitializationException {
+ final Map<PropertyDescriptor, String> properties = new LinkedHashMap<>();
+ properties.put(Kafka3ConnectionService.BOOTSTRAP_SERVERS, kafkaContainer.getBootstrapServers());
+ final MockConfigurationContext configurationContext = new MockConfigurationContext(properties, null, null);
+
+ final List<ConfigVerificationResult> results = service.verify(
+ configurationContext, runner.getLogger(), getAdminClientConfigProperties());
+
+ assertFalse(results.isEmpty());
+
+ final ConfigVerificationResult firstResult = results.iterator().next();
+ assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, firstResult.getOutcome());
+ assertNotNull(firstResult.getExplanation());
+ }
+
+ @Test
+ void testVerifyFailed() {
+ final Map<PropertyDescriptor, String> properties = new LinkedHashMap<>();
+ properties.put(Kafka3ConnectionService.BOOTSTRAP_SERVERS, UNREACHABLE_BOOTSTRAP_SERVERS);
+ properties.put(Kafka3ConnectionService.CLIENT_TIMEOUT, CLIENT_TIMEOUT);
+
+ final MockConfigurationContext configurationContext = new MockConfigurationContext(
+ properties, null, null);
+
+ final List<ConfigVerificationResult> results = service.verify(
+ configurationContext, runner.getLogger(), getAdminClientConfigProperties());
+
+ assertFalse(results.isEmpty());
+
+ final ConfigVerificationResult firstResult = results.iterator().next();
+ assertEquals(ConfigVerificationResult.Outcome.FAILED, firstResult.getOutcome());
+ }
+
+ @Test
+ void testGetProducerService() {
+ final ProducerConfiguration producerConfiguration = new ProducerConfiguration(false, null, null, null, null);
+ final KafkaProducerService producerService = service.getProducerService(producerConfiguration);
+ final List<PartitionState> partitionStates = producerService.getPartitionStates(TOPIC);
+ assertPartitionStatesFound(partitionStates);
+ }
+
+ @Test
+ void testGetConsumerService() {
+ final KafkaConsumerService consumerService = service.getConsumerService(null);
+ final PollingContext pollingContext = new PollingContext(
+ GROUP_ID, Collections.singleton(TOPIC), AutoOffsetReset.EARLIEST, Duration.ofSeconds(1));
+ final List<PartitionState> partitionStates = consumerService.getPartitionStates(pollingContext);
+ assertPartitionStatesFound(partitionStates);
+ }
+
+ private void assertPartitionStatesFound(final List<PartitionState> partitionStates) {
+ assertEquals(1, partitionStates.size());
+ final PartitionState partitionState = partitionStates.iterator().next();
+ assertEquals(TOPIC, partitionState.getTopic());
+ assertEquals(0, partitionState.getPartition());
+ }
+
+ private Iterator<ByteRecord> poll(final KafkaConsumerService consumerService, final PollingContext pollingContext) {
+ Iterator<ByteRecord> consumerRecords = Collections.emptyIterator();
+
+ for (int i = 0; i < POLLING_ATTEMPTS; i++) {
+ final Iterable<ByteRecord> records = consumerService.poll(pollingContext);
+ assertNotNull(records);
+ consumerRecords = records.iterator();
+ if (consumerRecords.hasNext()) {
+ break;
+ }
+ }
+
+ return consumerRecords;
+ }
+
+ protected String addSSLContextService(final TestRunner runner) throws InitializationException {
+ final String identifier = SSLContextService.class.getSimpleName();
+ final SSLContextService service = new StandardRestrictedSSLContextService();
+ runner.addControllerService(identifier, service);
+
+ // TemporaryKeyStoreBuilder sets keystorePassword and keyPassword to the same value.
+ // The SSL Context Service uses specified Keystore Password as the Key Password.
+ //runner.setProperty(service, StandardRestrictedSSLContextService.KEY_PASSWORD, tlsConfiguration.getKeyPassword());
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE, tlsConfiguration.getKeystorePath());
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE_PASSWORD, tlsConfiguration.getKeystorePassword());
+ runner.setProperty(service, StandardRestrictedSSLContextService.KEYSTORE_TYPE, tlsConfiguration.getKeystoreType().getType());
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE, tlsConfiguration.getTruststorePath());
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE_PASSWORD, tlsConfiguration.getTruststorePassword());
+ runner.setProperty(service, StandardRestrictedSSLContextService.TRUSTSTORE_TYPE, tlsConfiguration.getTruststoreType().getType());
+
+ runner.enableControllerService(service);
+ return identifier;
+ }
+
+ protected String getJaasConfigKafkaContainer(String userName, String password) {
+ return String.format("%s required user_%s=\"%s\";",
+ PlainLoginModule.class.getName(), userName, password);
+ }
+
+ protected String getJaasConfigKafkaClient(String userName, String password) {
+ return String.format("%s required username=\"%s\" password=\"%s\";",
+ PlainLoginModule.class.getName(), userName, password);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceSSLIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceSSLIT.java
new file mode 100644
index 0000000000..ce08ba03a8
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceSSLIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service;
+
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.nifi.kafka.shared.property.KafkaClientProperty;
+import org.apache.nifi.kafka.shared.property.SecurityProtocol;
+import org.apache.nifi.reporting.InitializationException;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class Kafka3ConnectionServiceSSLIT extends Kafka3ConnectionServiceBaseIT {
+
+ protected Map<String, String> getKafkaContainerConfigProperties() {
+ final Map<String, String> properties = new LinkedHashMap<>(super.getKafkaContainerConfigProperties());
+ properties.put("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "BROKER:SSL,PLAINTEXT:SSL");
+ properties.put("KAFKA_SSL_KEYSTORE_LOCATION", tlsConfiguration.getKeystorePath());
+ properties.put("KAFKA_SSL_KEYSTORE_TYPE", tlsConfiguration.getKeystoreType().getType());
+ properties.put("KAFKA_SSL_KEYSTORE_PASSWORD", tlsConfiguration.getKeystorePassword());
+ properties.put("KAFKA_SSL_KEY_PASSWORD", tlsConfiguration.getKeyPassword());
+ properties.put("KAFKA_SSL_TRUSTSTORE_LOCATION", tlsConfiguration.getTruststorePath());
+ properties.put("KAFKA_SSL_TRUSTSTORE_TYPE", tlsConfiguration.getTruststoreType().getType());
+ properties.put("KAFKA_SSL_TRUSTSTORE_PASSWORD", tlsConfiguration.getTruststorePassword());
+ properties.put("KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND", "false");
+ properties.put("KAFKA_SSL_CLIENT_AUTH", "required");
+ properties.put("KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", " ");
+ return properties;
+ }
+
+ @Override
+ protected Map<String, String> getKafkaServiceConfigProperties() throws InitializationException {
+ final Map<String, String> properties = new LinkedHashMap<>(super.getKafkaServiceConfigProperties());
+ properties.put(Kafka3ConnectionService.SECURITY_PROTOCOL.getName(), SecurityProtocol.SSL.name());
+ properties.put(Kafka3ConnectionService.SSL_CONTEXT_SERVICE.getName(), addSSLContextService(runner));
+ return properties;
+ }
+
+ @Override
+ protected Map<String, String> getAdminClientConfigProperties() {
+ final Map<String, String> properties = new LinkedHashMap<>(super.getAdminClientConfigProperties());
+ properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SSL.name());
+ properties.put(KafkaClientProperty.SSL_KEY_PASSWORD.getProperty(), tlsConfiguration.getKeystorePassword());
+ properties.put(KafkaClientProperty.SSL_KEYSTORE_LOCATION.getProperty(), tlsConfiguration.getKeystorePath());
+ properties.put(KafkaClientProperty.SSL_KEYSTORE_TYPE.getProperty(), tlsConfiguration.getKeystoreType().getType());
+ properties.put(KafkaClientProperty.SSL_KEYSTORE_PASSWORD.getProperty(), tlsConfiguration.getKeystorePassword());
+ properties.put(KafkaClientProperty.SSL_TRUSTSTORE_LOCATION.getProperty(), tlsConfiguration.getTruststorePath());
+ properties.put(KafkaClientProperty.SSL_TRUSTSTORE_TYPE.getProperty(), tlsConfiguration.getTruststoreType().getType());
+ properties.put(KafkaClientProperty.SSL_TRUSTSTORE_PASSWORD.getProperty(), tlsConfiguration.getTruststorePassword());
+ return properties;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceSaslPlaintextIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceSaslPlaintextIT.java
new file mode 100644
index 0000000000..69e06b5989
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/Kafka3ConnectionServiceSaslPlaintextIT.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service;
+
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.common.config.SaslConfigs;
+import org.apache.nifi.kafka.shared.property.SaslMechanism;
+import org.apache.nifi.kafka.shared.property.SecurityProtocol;
+import org.apache.nifi.reporting.InitializationException;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class Kafka3ConnectionServiceSaslPlaintextIT extends Kafka3ConnectionServiceBaseIT {
+
+ protected Map<String, String> getKafkaContainerConfigProperties() {
+ final Map<String, String> properties = new LinkedHashMap<>(super.getKafkaContainerConfigProperties());
+ properties.put("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "BROKER:PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT");
+ properties.put("KAFKA_LISTENER_NAME_PLAINTEXT_SASL_ENABLED_MECHANISMS", "PLAIN");
+ properties.put("KAFKA_LISTENER_NAME_PLAINTEXT_PLAIN_SASL_JAAS_CONFIG",
+ getJaasConfigKafkaContainer(TEST_USERNAME, TEST_PASSWORD));
+ return properties;
+ }
+
+ @Override
+ protected Map<String, String> getKafkaServiceConfigProperties() throws InitializationException {
+ final Map<String, String> properties = new LinkedHashMap<>(super.getKafkaServiceConfigProperties());
+ properties.put(Kafka3ConnectionService.SECURITY_PROTOCOL.getName(), SecurityProtocol.SASL_PLAINTEXT.name());
+ properties.put(Kafka3ConnectionService.SASL_MECHANISM.getName(), SaslMechanism.PLAIN.getValue());
+ properties.put(Kafka3ConnectionService.SASL_USERNAME.getName(), TEST_USERNAME);
+ properties.put(Kafka3ConnectionService.SASL_PASSWORD.getName(), TEST_PASSWORD);
+ return properties;
+ }
+
+ @Override
+ protected Map<String, String> getAdminClientConfigProperties() {
+ final Map<String, String> properties = new LinkedHashMap<>(super.getAdminClientConfigProperties());
+ properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_PLAINTEXT.name());
+ properties.put(SaslConfigs.SASL_MECHANISM, SaslMechanism.PLAIN.getValue());
+ properties.put(SaslConfigs.SASL_JAAS_CONFIG, getJaasConfigKafkaClient(TEST_USERNAME, TEST_PASSWORD));
+ return properties;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/KafkaConnectivityIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/KafkaConnectivityIT.java
new file mode 100644
index 0000000000..0def454af5
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/KafkaConnectivityIT.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public class KafkaConnectivityIT {
+ private final Logger logger = LoggerFactory.getLogger(getClass());
+
+ private static final String TEST_TOPIC = "nifi-" + System.currentTimeMillis();
+
+ private static KafkaContainer kafka;
+
+ @BeforeAll
+ static void beforeAll() {
+ kafka = new KafkaContainer(DockerImageName.parse(Kafka3ConnectionServiceBaseIT.IMAGE_NAME));
+ kafka.start();
+ }
+
+ @AfterAll
+ static void afterAll() {
+ kafka.stop();
+ }
+
+ @Test
+ public void test_1_KafkaTestContainerProduceOne() throws ExecutionException, InterruptedException {
+ final Properties properties = new Properties();
+ properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
+ properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ try (KafkaProducer<String, String> producer = new KafkaProducer<>(properties)) {
+ final ProducerRecord<String, String> record = new ProducerRecord<>(TEST_TOPIC, "my-key", "my-value");
+ final Future<RecordMetadata> future = producer.send(record);
+ final RecordMetadata metadata = future.get();
+ assertEquals(TEST_TOPIC, metadata.topic());
+ logger.trace("OFFSET={} TIMESTAMP={}",
+ metadata.hasOffset() ? metadata.offset() : "NO-OFFSET",
+ metadata.hasTimestamp() ? new Date(metadata.timestamp()).toInstant().toString() : "NO-TIMESTAMP");
+ }
+ }
+
+ @Test
+ public void test_2_KafkaTestContainerConsumeOne() {
+ final Properties properties = new Properties();
+ properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
+ properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, Boolean.TRUE.toString());
+ properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
+ properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(properties)) {
+ consumer.subscribe(Collections.singletonList(TEST_TOPIC));
+ final ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
+ assertEquals(1, records.count());
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerPooledObjectFactoryTest.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerPooledObjectFactoryTest.java
new file mode 100644
index 0000000000..5833b0067f
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/java/org/apache/nifi/kafka/service/consumer/pool/ConsumerPooledObjectFactoryTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.service.consumer.pool;
+
+import org.apache.commons.pool2.PooledObject;
+import org.apache.commons.pool2.impl.DefaultPooledObject;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Properties;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyCollection;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class ConsumerPooledObjectFactoryTest {
+ private static final String GROUP_ID = ConsumerPooledObjectFactoryTest.class.getSimpleName();
+
+ private static final String TOPIC = String.class.getSimpleName();
+
+ @Mock
+ ConsumerFactory consumerFactory;
+
+ @Mock
+ Consumer<byte[], byte[]> mockConsumer;
+
+ ConsumerPooledObjectFactory factory;
+
+ @BeforeEach
+ void setFactory() {
+ final Properties properties = new Properties();
+ factory = new ConsumerPooledObjectFactory(properties, consumerFactory);
+ }
+
+ @Test
+ void testCreate() {
+ final Collection<String> topics = Collections.singleton(TOPIC);
+ final Subscription subscription = new Subscription(GROUP_ID, topics, AutoOffsetReset.EARLIEST);
+
+ when(consumerFactory.newConsumer(any(Properties.class))).thenReturn(mockConsumer);
+ final Consumer<byte[], byte[]> consumer = factory.create(subscription);
+
+ assertNotNull(consumer);
+
+ verify(mockConsumer).subscribe(anyCollection());
+ }
+
+ @Test
+ void testWrap() {
+ final PooledObject<Consumer<byte[], byte[]>> pooledObject = factory.wrap(mockConsumer);
+
+ assertNotNull(pooledObject);
+ }
+
+ @Test
+ void testDestroyObject() {
+ final PooledObject<Consumer<byte[], byte[]>> pooledObject = new DefaultPooledObject<>(mockConsumer);
+
+ final Subscription subscription = new Subscription(GROUP_ID, Collections.singleton(TOPIC), AutoOffsetReset.EARLIEST);
+ factory.destroyObject(subscription, pooledObject);
+
+ verify(mockConsumer).close();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/resources/simplelogger.properties b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/resources/simplelogger.properties
new file mode 100644
index 0000000000..c3e6201320
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-service/src/test/resources/simplelogger.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS
+
+org.slf4j.simpleLogger.log=WARN
+org.slf4j.simpleLogger.log.org.apache.nifi.kafka.service.it=WARN
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/README.md b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/README.md
new file mode 100644
index 0000000000..46a1287a59
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/README.md
@@ -0,0 +1,20 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Generate Jacoco Report
+- terminal command
+ - `/nifi/nifi-nar-bundles/nifi-kafka-bundle % mvn clean test verify -P integration-tests -P integration-tests-ci`
+- output written to
+ - `nifi/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/target/site/jacoco-aggregate/index.html`
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/pom.xml
new file mode 100644
index 0000000000..2995f15541
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-code-coverage/pom.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-code-coverage</artifactId>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>report-aggregate</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>report-aggregate</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>merge-results</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>merge</goal>
+ </goals>
+ <configuration>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <includes>
+ <include>**/target/jacoco.exec</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <destFile>${project.basedir}/target/aggregate.exec</destFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-shared</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-2-6-processors</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-2-6-integration</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-service</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-processors</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-integration</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-nar/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-nar/pom.xml
new file mode 100644
index 0000000000..98cf2fa96e
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-nar/pom.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-processors</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api-nar</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml
new file mode 100644
index 0000000000..7f52b5d539
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-bundle</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-processors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-shared</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-service-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
new file mode 100644
index 0000000000..10b7c86bba
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
@@ -0,0 +1,477 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.kafka.processors.common.KafkaUtils;
+import org.apache.nifi.kafka.processors.consumer.OffsetTracker;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.consumer.bundle.ByteRecordBundler;
+import org.apache.nifi.kafka.processors.consumer.convert.FlowFileStreamKafkaMessageConverter;
+import org.apache.nifi.kafka.processors.consumer.convert.KafkaMessageConverter;
+import org.apache.nifi.kafka.processors.consumer.convert.RecordStreamKafkaMessageConverter;
+import org.apache.nifi.kafka.processors.consumer.convert.WrapperRecordStreamKafkaMessageConverter;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.service.api.consumer.ConsumerConfiguration;
+import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
+import org.apache.nifi.kafka.service.api.consumer.PollingContext;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+import org.apache.nifi.kafka.shared.property.KeyFormat;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.VerifiableProcessor;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.util.StringUtils;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+
+@CapabilityDescription("Consumes messages from Apache Kafka Consumer API. "
+ + "The complementary NiFi processor for sending messages is PublishKafka. The Processor supports consumption of Kafka messages, optionally interpreted as NiFi records. "
+ + "Please note that, at this time (in read record mode), the Processor assumes that "
+ + "all records that are retrieved from a given partition have the same schema. For this mode, if any of the Kafka messages are pulled but cannot be parsed or written with the "
+ + "configured Record Reader or Record Writer, the contents of the message will be written to a separate FlowFile, and that FlowFile will be transferred to the "
+ + "'parse.failure' relationship. Otherwise, each FlowFile is sent to the 'success' relationship and may contain many individual messages within the single FlowFile. "
+ + "A 'record.count' attribute is added to indicate how many messages are contained in the FlowFile. No two Kafka messages will be placed into the same FlowFile if they "
+ + "have different schemas, or if they have different values for a message header that is included by the <Headers to Add as Attributes> property.")
+@Tags({"Kafka", "Get", "Record", "csv", "avro", "json", "Ingest", "Ingress", "Topic", "PubSub", "Consume"})
+@WritesAttributes({
+ @WritesAttribute(attribute = "record.count", description = "The number of records received"),
+ @WritesAttribute(attribute = "mime.type", description = "The MIME Type that is provided by the configured Record Writer"),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_COUNT, description = "The number of messages written if more than one"),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_KEY, description = "The key of message if present and if single message. "
+ + "How the key is encoded depends on the value of the 'Key Attribute Encoding' property."),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_OFFSET, description = "The offset of the message in the partition of the topic."),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_TIMESTAMP, description = "The timestamp of the message in the partition of the topic."),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_PARTITION, description = "The partition of the topic the message or message bundle is from"),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_TOPIC, description = "The topic the message or message bundle is from"),
+ @WritesAttribute(attribute = KafkaFlowFileAttribute.KAFKA_TOMBSTONE, description = "Set to true if the consumed message is a tombstone message")
+})
+@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN)
+@SeeAlso({PublishKafka.class})
+public class ConsumeKafka extends AbstractProcessor implements VerifiableProcessor {
+
+ static final AllowableValue TOPIC_NAME = new AllowableValue("names", "names", "Topic is a full topic name or comma separated list of names");
+ static final AllowableValue TOPIC_PATTERN = new AllowableValue("pattern", "pattern", "Topic is a regular expression according to the Java Pattern syntax");
+
+ static final PropertyDescriptor CONNECTION_SERVICE = new PropertyDescriptor.Builder()
+ .name("Kafka Connection Service")
+ .description("Provides connections to Kafka Broker for publishing Kafka Records")
+ .identifiesControllerService(KafkaConnectionService.class)
+ .expressionLanguageSupported(NONE)
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor GROUP_ID = new PropertyDescriptor.Builder()
+ .name("Group ID")
+ .description("Kafka Consumer Group Identifier corresponding to Kafka group.id property")
+ .required(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .build();
+
+ static final PropertyDescriptor TOPIC_FORMAT = new PropertyDescriptor.Builder()
+ .name("Topic Format")
+ .description("Specifies whether the Topics provided are a comma separated list of names or a single regular expression")
+ .required(true)
+ .allowableValues(TOPIC_NAME, TOPIC_PATTERN)
+ .defaultValue(TOPIC_NAME)
+ .build();
+
+ static final PropertyDescriptor TOPICS = new PropertyDescriptor.Builder()
+ .name("Topics")
+ .description("The name or pattern of the Kafka Topics from which the Processor consumes Kafka Records. More than one can be supplied if comma separated.")
+ .required(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
+ .build();
+
+ static final PropertyDescriptor AUTO_OFFSET_RESET = new PropertyDescriptor.Builder()
+ .name("auto.offset.reset")
+ .displayName("Auto Offset Reset")
+ .description("Automatic offset configuration applied when no previous consumer offset found corresponding to Kafka auto.offset.reset property")
+ .required(true)
+ .allowableValues(AutoOffsetReset.class)
+ .defaultValue(AutoOffsetReset.LATEST.getValue())
+ .expressionLanguageSupported(NONE)
+ .build();
+
+ static final PropertyDescriptor COMMIT_OFFSETS = new PropertyDescriptor.Builder()
+ .name("Commit Offsets")
+ .description("Specifies whether this Processor should commit the offsets to Kafka after receiving messages. Typically, this value should be set to true " +
+ "so that messages that are received are not duplicated. However, in certain scenarios, we may want to avoid committing the offsets, that the data can be " +
+ "processed and later acknowledged by PublishKafka in order to provide Exactly Once semantics.")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .build();
+
+ static final PropertyDescriptor MAX_UNCOMMITTED_TIME = new PropertyDescriptor.Builder()
+ .name("Max Uncommitted Time")
+ .description("Specifies the maximum amount of time allowed to pass before offsets must be committed. "
+ + "This value impacts how often offsets will be committed. Committing offsets less often increases "
+ + "throughput but also increases the window of potential data duplication in the event of a rebalance "
+ + "or JVM restart between commits. This value is also related to maximum poll records and the use "
+ + "of a message demarcator. When using a message demarcator we can have far more uncommitted messages "
+ + "than when we're not as there is much less for us to keep track of in memory.")
+ .required(true)
+ .defaultValue("1 s")
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .dependsOn(COMMIT_OFFSETS, "true")
+ .build();
+
+ static final PropertyDescriptor HEADER_ENCODING = new PropertyDescriptor.Builder()
+ .name("Header Encoding")
+ .description("Character encoding applied when reading Kafka Record Header values and writing FlowFile attributes")
+ .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
+ .defaultValue(StandardCharsets.UTF_8.name())
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor HEADER_NAME_PATTERN = new PropertyDescriptor.Builder()
+ .name("Header Name Pattern")
+ .description("Regular Expression Pattern applied to Kafka Record Header Names for selecting Header Values to be written as FlowFile attributes")
+ .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .required(false)
+ .build();
+
+ static final PropertyDescriptor PROCESSING_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Processing Strategy")
+ .description("Strategy for processing Kafka Records and writing serialized output to FlowFiles")
+ .required(true)
+ .allowableValues(ProcessingStrategy.class)
+ .defaultValue(ProcessingStrategy.FLOW_FILE.getValue())
+ .expressionLanguageSupported(NONE)
+ .build();
+
+ static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder()
+ .name("Record Reader")
+ .description("The Record Reader to use for incoming Kafka messages")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(true)
+ .dependsOn(PROCESSING_STRATEGY, ProcessingStrategy.RECORD)
+ .build();
+
+ static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder()
+ .name("Record Writer")
+ .description("The Record Writer to use in order to serialize the outgoing FlowFiles")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .required(true)
+ .dependsOn(PROCESSING_STRATEGY, ProcessingStrategy.RECORD)
+ .build();
+
+ static final PropertyDescriptor OUTPUT_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Output Strategy")
+ .description("The format used to output the Kafka Record into a FlowFile Record.")
+ .required(true)
+ .defaultValue(OutputStrategy.USE_VALUE)
+ .allowableValues(OutputStrategy.class)
+ .dependsOn(PROCESSING_STRATEGY, ProcessingStrategy.RECORD)
+ .build();
+
+ static final PropertyDescriptor KEY_ATTRIBUTE_ENCODING = new PropertyDescriptor.Builder()
+ .name("Key Attribute Encoding")
+ .description("Encoding for value of configured FlowFile attribute containing Kafka Record Key.")
+ .required(true)
+ .defaultValue(KeyEncoding.UTF8)
+ .allowableValues(KeyEncoding.class)
+ .dependsOn(OUTPUT_STRATEGY, OutputStrategy.USE_VALUE)
+ .build();
+
+ static final PropertyDescriptor KEY_FORMAT = new PropertyDescriptor.Builder()
+ .name("Key Format")
+ .description("Specifies how to represent the Kafka Record Key in the output FlowFile")
+ .required(true)
+ .defaultValue(KeyFormat.BYTE_ARRAY)
+ .allowableValues(KeyFormat.class)
+ .dependsOn(OUTPUT_STRATEGY, OutputStrategy.USE_WRAPPER)
+ .build();
+
+ static final PropertyDescriptor KEY_RECORD_READER = new PropertyDescriptor.Builder()
+ .name("Key Record Reader")
+ .description("The Record Reader to use for parsing the Kafka Record Key into a Record")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .required(true)
+ .dependsOn(KEY_FORMAT, KeyFormat.RECORD)
+ .build();
+
+ static final PropertyDescriptor MESSAGE_DEMARCATOR = new PropertyDescriptor.Builder()
+ .name("Message Demarcator")
+ .required(true)
+ .addValidator(Validator.VALID)
+ .description("Since KafkaConsumer receives messages in batches, this Processor has an option to output FlowFiles which contains "
+ + "all Kafka messages in a single batch for a given topic and partition and this property allows you to provide a string (interpreted as UTF-8) to use "
+ + "for demarcating apart multiple Kafka messages. This is an optional property and if not provided each Kafka message received "
+ + "will result in a single FlowFile which "
+ + "time it is triggered. To enter special character such as 'new line' use CTRL+Enter or Shift+Enter depending on the OS")
+ .dependsOn(PROCESSING_STRATEGY, ProcessingStrategy.DEMARCATOR)
+ .build();
+
+ static final PropertyDescriptor SEPARATE_BY_KEY = new PropertyDescriptor.Builder()
+ .name("Separate By Key")
+ .description("When this property is enabled, two messages will only be added to the same FlowFile if both of the Kafka Messages have identical keys.")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("false")
+ .dependsOn(MESSAGE_DEMARCATOR)
+ .build();
+
+ public static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("FlowFiles containing one or more serialized Kafka Records")
+ .build();
+
+ private static final List<PropertyDescriptor> DESCRIPTORS = List.of(
+ CONNECTION_SERVICE,
+ GROUP_ID,
+ TOPIC_FORMAT,
+ TOPICS,
+ AUTO_OFFSET_RESET,
+ COMMIT_OFFSETS,
+ MAX_UNCOMMITTED_TIME,
+ HEADER_NAME_PATTERN,
+ HEADER_ENCODING,
+ PROCESSING_STRATEGY,
+ RECORD_READER,
+ RECORD_WRITER,
+ OUTPUT_STRATEGY,
+ KEY_ATTRIBUTE_ENCODING,
+ KEY_FORMAT,
+ KEY_RECORD_READER,
+ MESSAGE_DEMARCATOR,
+ SEPARATE_BY_KEY
+ );
+
+ private static final Set<Relationship> RELATIONSHIPS = Collections.singleton(SUCCESS);
+
+ private KafkaConsumerService consumerService;
+
+ private Charset headerEncoding;
+
+ private Pattern headerNamePattern;
+
+ private KeyEncoding keyEncoding;
+
+ private OutputStrategy outputStrategy;
+
+ private KeyFormat keyFormat;
+
+ private boolean commitOffsets;
+
+ @Override
+ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return DESCRIPTORS;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void onScheduled(final ProcessContext context) {
+ final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
+ consumerService = connectionService.getConsumerService(new ConsumerConfiguration());
+ headerEncoding = Charset.forName(context.getProperty(HEADER_ENCODING).getValue());
+
+ final String headerNamePatternProperty = context.getProperty(HEADER_NAME_PATTERN).getValue();
+ if (StringUtils.isNotBlank(headerNamePatternProperty)) {
+ headerNamePattern = Pattern.compile(headerNamePatternProperty);
+ } else {
+ headerNamePattern = null;
+ }
+ keyEncoding = context.getProperty(KEY_ATTRIBUTE_ENCODING).asAllowableValue(KeyEncoding.class);
+ commitOffsets = context.getProperty(COMMIT_OFFSETS).asBoolean();
+ outputStrategy = context.getProperty(OUTPUT_STRATEGY).asAllowableValue(OutputStrategy.class);
+ keyFormat = context.getProperty(KEY_FORMAT).asAllowableValue(KeyFormat.class);
+ }
+
+ @OnStopped
+ public void onStopped() {
+ // discard reference; leave controller service state intact
+ consumerService = null;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ final PollingContext pollingContext = getPollingContext(context);
+
+ final Iterator<ByteRecord> consumerRecords = consumerService.poll(pollingContext).iterator();
+ if (consumerRecords.hasNext()) {
+ processConsumerRecords(context, session, pollingContext, consumerRecords);
+ } else {
+ getLogger().debug("No Kafka Records consumed: {}", pollingContext);
+ context.yield();
+ }
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verify(final ProcessContext context, final ComponentLog verificationLogger, final Map<String, String> attributes) {
+ final List<ConfigVerificationResult> verificationResults = new ArrayList<>();
+
+ final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
+ final KafkaConsumerService consumerService = connectionService.getConsumerService(new ConsumerConfiguration());
+
+ final ConfigVerificationResult.Builder verificationPartitions = new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Topic Partitions");
+
+ final PollingContext pollingContext = getPollingContext(context);
+
+ try {
+ final List<PartitionState> partitionStates = consumerService.getPartitionStates(pollingContext);
+ verificationPartitions
+ .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .explanation(String.format("Partitions [%d] found for Topics %s", partitionStates.size(), pollingContext.getTopics()));
+ } catch (final Exception e) {
+ getLogger().error("Topics {} Partition verification failed", pollingContext.getTopics(), e);
+ verificationPartitions
+ .outcome(ConfigVerificationResult.Outcome.FAILED)
+ .explanation(String.format("Topics %s Partition access failed: %s", pollingContext.getTopics(), e));
+ }
+ verificationResults.add(verificationPartitions.build());
+
+ return verificationResults;
+ }
+
+ private void processConsumerRecords(final ProcessContext context, final ProcessSession session, final PollingContext pollingContext, final Iterator<ByteRecord> consumerRecords) {
+ final ProcessingStrategy processingStrategy = ProcessingStrategy.valueOf(context.getProperty(PROCESSING_STRATEGY).getValue());
+ // model this switch on the existing implementation at `ConsumerLease.processRecords()`
+ if (ProcessingStrategy.FLOW_FILE == processingStrategy) {
+ processInputFlowFile(session, pollingContext, consumerRecords);
+ } else if (ProcessingStrategy.DEMARCATOR == processingStrategy) {
+ final Iterator<ByteRecord> iteratorDemarcator = transformDemarcator(context, consumerRecords);
+ processInputFlowFile(session, pollingContext, iteratorDemarcator);
+ } else if (ProcessingStrategy.RECORD == processingStrategy) {
+ processInputRecords(context, session, pollingContext, consumerRecords);
+ } else {
+ throw new ProcessException(String.format("Processing Strategy not supported [%s]", processingStrategy));
+ }
+ }
+
+ private Iterator<ByteRecord> transformDemarcator(final ProcessContext context, final Iterator<ByteRecord> consumerRecords) {
+ final PropertyValue propertyValueDemarcator = context.getProperty(ConsumeKafka.MESSAGE_DEMARCATOR);
+ if (propertyValueDemarcator.isSet()) {
+ final byte[] demarcator = propertyValueDemarcator.getValue().getBytes(StandardCharsets.UTF_8);
+ final boolean separateByKey = context.getProperty(SEPARATE_BY_KEY).asBoolean();
+ return new ByteRecordBundler(demarcator, separateByKey, keyEncoding, headerNamePattern, headerEncoding, commitOffsets).bundle(consumerRecords);
+ } else {
+ return consumerRecords;
+ }
+ }
+
+ private void processInputRecords(final ProcessContext context, final ProcessSession session, final PollingContext pollingContext, final Iterator<ByteRecord> consumerRecords) {
+ if (OutputStrategy.USE_VALUE.equals(outputStrategy)) {
+ processOutputStrategyUseValue(context, session, pollingContext, consumerRecords);
+ } else if (OutputStrategy.USE_WRAPPER.equals(outputStrategy)) {
+ processOutputStrategyUseWrapper(context, session, pollingContext, consumerRecords);
+ } else {
+ throw new ProcessException(String.format("Output Strategy not supported [%s]", outputStrategy));
+ }
+ }
+
+ private void processOutputStrategyUseWrapper(final ProcessContext context, final ProcessSession session, final PollingContext pollingContext, final Iterator<ByteRecord> consumerRecords) {
+ final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+ final RecordReaderFactory keyReaderFactory = context.getProperty(KEY_RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final OffsetTracker offsetTracker = new OffsetTracker();
+ final Runnable onSuccess = commitOffsets
+ ? () -> session.commitAsync(() -> consumerService.commit(offsetTracker.getPollingSummary(pollingContext)))
+ : session::commitAsync;
+ final KafkaMessageConverter converter = new WrapperRecordStreamKafkaMessageConverter(readerFactory, writerFactory, keyReaderFactory,
+ headerEncoding, headerNamePattern, keyFormat, keyEncoding, commitOffsets, offsetTracker, onSuccess, getLogger());
+ converter.toFlowFiles(session, consumerRecords);
+ }
+
+ private void processOutputStrategyUseValue(final ProcessContext context, final ProcessSession session, final PollingContext pollingContext, final Iterator<ByteRecord> consumerRecords) {
+ final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+ final OffsetTracker offsetTracker = new OffsetTracker();
+ final Runnable onSuccess = commitOffsets
+ ? () -> session.commitAsync(() -> consumerService.commit(offsetTracker.getPollingSummary(pollingContext)))
+ : session::commitAsync;
+ final KafkaMessageConverter converter = new RecordStreamKafkaMessageConverter(readerFactory, writerFactory,
+ headerEncoding, headerNamePattern, keyEncoding, commitOffsets, offsetTracker, onSuccess, getLogger());
+ converter.toFlowFiles(session, consumerRecords);
+ }
+
+ private void processInputFlowFile(final ProcessSession session, final PollingContext pollingContext, final Iterator<ByteRecord> consumerRecords) {
+ final OffsetTracker offsetTracker = new OffsetTracker();
+ final Runnable onSuccess = commitOffsets
+ ? () -> session.commitAsync(() -> consumerService.commit(offsetTracker.getPollingSummary(pollingContext)))
+ : session::commitAsync;
+ final KafkaMessageConverter converter = new FlowFileStreamKafkaMessageConverter(
+ headerEncoding, headerNamePattern, keyEncoding, commitOffsets, offsetTracker, onSuccess);
+ converter.toFlowFiles(session, consumerRecords);
+ }
+
+ private PollingContext getPollingContext(final ProcessContext context) {
+ final String groupId = context.getProperty(GROUP_ID).getValue();
+ final String offsetReset = context.getProperty(AUTO_OFFSET_RESET).getValue();
+ final AutoOffsetReset autoOffsetReset = AutoOffsetReset.valueOf(offsetReset.toUpperCase());
+ final String topics = context.getProperty(TOPICS).evaluateAttributeExpressions().getValue();
+ final String topicFormat = context.getProperty(TOPIC_FORMAT).getValue();
+ final Duration maxUncommittedTime = context.getProperty(MAX_UNCOMMITTED_TIME).asDuration();
+
+ final PollingContext pollingContext;
+ if (topicFormat.equals(TOPIC_PATTERN.getValue())) {
+ final Pattern topicPattern = Pattern.compile(topics.trim());
+ pollingContext = new PollingContext(groupId, topicPattern, autoOffsetReset, maxUncommittedTime);
+ } else if (topicFormat.equals(TOPIC_NAME.getValue())) {
+ final Collection<String> topicList = KafkaUtils.toTopicList(topics);
+ pollingContext = new PollingContext(groupId, topicList, autoOffsetReset, maxUncommittedTime);
+ } else {
+ throw new ProcessException(String.format("Topic Format [%s] not supported", topicFormat));
+ }
+ return pollingContext;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/PublishKafka.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/PublishKafka.java
new file mode 100644
index 0000000000..4c41bcd246
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/PublishKafka.java
@@ -0,0 +1,536 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.kafka.processors.producer.PartitionStrategy;
+import org.apache.nifi.kafka.processors.producer.common.PublishKafkaUtil;
+import org.apache.nifi.kafka.processors.producer.config.DeliveryGuarantee;
+import org.apache.nifi.kafka.processors.producer.convert.DelimitedStreamKafkaRecordConverter;
+import org.apache.nifi.kafka.processors.producer.convert.FlowFileStreamKafkaRecordConverter;
+import org.apache.nifi.kafka.processors.producer.convert.KafkaRecordConverter;
+import org.apache.nifi.kafka.processors.producer.convert.RecordStreamKafkaRecordConverter;
+import org.apache.nifi.kafka.processors.producer.convert.RecordWrapperStreamKafkaRecordConverter;
+import org.apache.nifi.kafka.processors.producer.header.AttributesHeadersFactory;
+import org.apache.nifi.kafka.processors.producer.header.HeadersFactory;
+import org.apache.nifi.kafka.processors.producer.key.AttributeKeyFactory;
+import org.apache.nifi.kafka.processors.producer.key.KeyFactory;
+import org.apache.nifi.kafka.processors.producer.key.MessageKeyFactory;
+import org.apache.nifi.kafka.processors.producer.wrapper.RecordMetadataStrategy;
+import org.apache.nifi.kafka.service.api.KafkaConnectionService;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.producer.FlowFileResult;
+import org.apache.nifi.kafka.service.api.producer.KafkaProducerService;
+import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration;
+import org.apache.nifi.kafka.service.api.producer.PublishContext;
+import org.apache.nifi.kafka.service.api.producer.RecordSummary;
+import org.apache.nifi.kafka.service.api.record.KafkaRecord;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.kafka.shared.component.KafkaPublishComponent;
+import org.apache.nifi.kafka.shared.property.FailureStrategy;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+import org.apache.nifi.kafka.shared.property.PublishStrategy;
+import org.apache.nifi.kafka.shared.transaction.TransactionIdSupplier;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.VerifiableProcessor;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+
+import java.io.BufferedInputStream;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+
+@Tags({"Apache", "Kafka", "Record", "csv", "json", "avro", "logs", "Put", "Send", "Message", "PubSub"})
+@CapabilityDescription("Sends the contents of a FlowFile as either a message or as individual records to Apache Kafka using the Kafka Producer API. "
+ + "The messages to send may be individual FlowFiles, may be delimited using a "
+ + "user-specified delimiter (such as a new-line), or "
+ + "may be record-oriented data that can be read by the configured Record Reader. "
+ + "The complementary NiFi processor for fetching messages is ConsumeKafka.")
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+@ReadsAttribute(attribute = KafkaFlowFileAttribute.KAFKA_TOMBSTONE, description = "If this attribute is set to 'true', if the processor is not configured "
+ + "with a demarcator and if the FlowFile's content is null, then a tombstone message with zero bytes will be sent to Kafka.")
+@WritesAttribute(attribute = "msg.count", description = "The number of messages that were sent to Kafka for this FlowFile. This attribute is added only to "
+ + "FlowFiles that are routed to success.")
+@SeeAlso({ConsumeKafka.class})
+public class PublishKafka extends AbstractProcessor implements KafkaPublishComponent, VerifiableProcessor {
+ protected static final String MSG_COUNT = "msg.count";
+
+ public static final PropertyDescriptor CONNECTION_SERVICE = new PropertyDescriptor.Builder()
+ .name("Kafka Connection Service")
+ .description("Provides connections to Kafka Broker for publishing Kafka Records")
+ .identifiesControllerService(KafkaConnectionService.class)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .required(true)
+ .build();
+
+ public static final PropertyDescriptor TOPIC_NAME = new PropertyDescriptor.Builder()
+ .name("Topic Name")
+ .description("Name of the Kafka Topic to which the Processor publishes Kafka Records")
+ .required(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .build();
+
+ static final PropertyDescriptor DELIVERY_GUARANTEE = new PropertyDescriptor.Builder()
+ .name("acks")
+ .displayName("Delivery Guarantee")
+ .description("Specifies the requirement for guaranteeing that a message is sent to Kafka. Corresponds to Kafka Client acks property.")
+ .required(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .allowableValues(DeliveryGuarantee.class)
+ .defaultValue(DeliveryGuarantee.DELIVERY_REPLICATED)
+ .build();
+
+ static final PropertyDescriptor COMPRESSION_CODEC = new PropertyDescriptor.Builder()
+ .name("compression.type")
+ .displayName("Compression Type")
+ .description("Specifies the compression strategy for records sent to Kafka. Corresponds to Kafka Client compression.type property.")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .allowableValues("none", "gzip", "snappy", "lz4")
+ .defaultValue("none")
+ .build();
+
+ public static final PropertyDescriptor MAX_REQUEST_SIZE = new PropertyDescriptor.Builder()
+ .name("max.request.size")
+ .displayName("Max Request Size")
+ .description("The maximum size of a request in bytes. Corresponds to Kafka Client max.request.size property.")
+ .required(true)
+ .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .defaultValue("1 MB")
+ .build();
+
+ public static final PropertyDescriptor TRANSACTIONS_ENABLED = new PropertyDescriptor.Builder()
+ .name("Transactions Enabled")
+ .description("Specifies whether to provide transactional guarantees when communicating with Kafka. If there is a problem sending data to Kafka, "
+ + "and this property is set to false, then the messages that have already been sent to Kafka will continue on and be delivered to consumers. "
+ + "If this is set to true, then the Kafka transaction will be rolled back so that those messages are not available to consumers. Setting this to true "
+ + "requires that the [Delivery Guarantee] property be set to [Guarantee Replicated Delivery.]")
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor TRANSACTIONAL_ID_PREFIX = new PropertyDescriptor.Builder()
+ .name("Transactional ID Prefix")
+ .description("Specifies the KafkaProducer config transactional.id will be a generated UUID and will be prefixed with the configured string.")
+ .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
+ .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
+ .dependsOn(TRANSACTIONS_ENABLED, "true")
+ .required(false)
+ .build();
+
+ static final PropertyDescriptor PARTITION_CLASS = new PropertyDescriptor.Builder()
+ .name("partitioner.class")
+ .displayName("Partitioner Class")
+ .description("Specifies which class to use to compute a partition id for a message. Corresponds to Kafka Client partitioner.class property.")
+ .allowableValues(PartitionStrategy.class)
+ .defaultValue(PartitionStrategy.RANDOM_PARTITIONING.getValue())
+ .required(true)
+ .build();
+
+ public static final PropertyDescriptor PARTITION = new PropertyDescriptor.Builder()
+ .name("partition")
+ .displayName("Partition")
+ .description("Specifies the Kafka Partition destination for Records.")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .build();
+
+ static final PropertyDescriptor MESSAGE_DEMARCATOR = new PropertyDescriptor.Builder()
+ .name("Message Demarcator")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .description("Specifies the string (interpreted as UTF-8) to use for demarcating multiple messages within "
+ + "a single FlowFile. If not specified, the entire content of the FlowFile will be used as a single message. If specified, the "
+ + "contents of the FlowFile will be split on this delimiter and each section sent as a separate Kafka message. "
+ + "To enter special character such as 'new line' use CTRL+Enter or Shift+Enter, depending on your OS.")
+ .build();
+
+ static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder()
+ .name("Record Reader")
+ .description("The Record Reader to use for incoming FlowFiles")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .build();
+
+ static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder()
+ .name("Record Writer")
+ .description("The Record Writer to use in order to serialize the data before sending to Kafka")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .build();
+
+ public static final PropertyDescriptor PUBLISH_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Publish Strategy")
+ .description("The format used to publish the incoming FlowFile record to Kafka.")
+ .required(true)
+ .defaultValue(PublishStrategy.USE_VALUE)
+ .allowableValues(PublishStrategy.class)
+ .build();
+
+ public static final PropertyDescriptor MESSAGE_KEY_FIELD = new PropertyDescriptor.Builder()
+ .name("Message Key Field")
+ .description("The name of a field in the Input Records that should be used as the Key for the Kafka message.")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(PUBLISH_STRATEGY, PublishStrategy.USE_VALUE)
+ .required(false)
+ .build();
+
+ public static final PropertyDescriptor ATTRIBUTE_HEADER_PATTERN = new PropertyDescriptor.Builder()
+ .name("FlowFile Attribute Header Pattern")
+ .description("A Regular Expression that is matched against all FlowFile attribute names. "
+ + "Any attribute whose name matches the pattern will be added to the Kafka messages as a Header. "
+ + "If not specified, no FlowFile attributes will be added as headers.")
+ .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .dependsOn(PUBLISH_STRATEGY, PublishStrategy.USE_VALUE)
+ .required(false)
+ .build();
+
+ static final PropertyDescriptor HEADER_ENCODING = new PropertyDescriptor.Builder()
+ .name("Header Encoding")
+ .description("For any attribute that is added as a Kafka Record Header, this property indicates the Character Encoding to use for serializing the headers.")
+ .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
+ .defaultValue(StandardCharsets.UTF_8.displayName())
+ .required(true)
+ .dependsOn(ATTRIBUTE_HEADER_PATTERN)
+ .build();
+
+ static final PropertyDescriptor KAFKA_KEY = new PropertyDescriptor.Builder()
+ .name("Kafka Key")
+ .description("The Key to use for the Message. "
+ + "If not specified, the FlowFile attribute 'kafka.key' is used as the message key, if it is present."
+ + "Beware that setting Kafka key and demarcating at the same time may potentially lead to many Kafka messages with the same key."
+ + "Normally this is not a problem as Kafka does not enforce or assume message and key uniqueness. Still, setting the demarcator and Kafka key at the same time poses a risk of "
+ + "data loss on Kafka. During a topic compaction on Kafka, messages will be deduplicated based on this key.")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER)
+ .build();
+
+ static final PropertyDescriptor KEY_ATTRIBUTE_ENCODING = new PropertyDescriptor.Builder()
+ .name("Kafka Key Attribute Encoding")
+ .description("FlowFiles that are emitted have an attribute named '" + KafkaFlowFileAttribute.KAFKA_KEY + "'. This property dictates how the value of the attribute should be encoded.")
+ .required(true)
+ .defaultValue(KeyEncoding.UTF8.getValue())
+ .allowableValues(KeyEncoding.class)
+ .dependsOn(PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER)
+ .build();
+
+ static final PropertyDescriptor RECORD_KEY_WRITER = new PropertyDescriptor.Builder()
+ .name("Record Key Writer")
+ .description("The Record Key Writer to use for outgoing FlowFiles")
+ .required(false)
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER)
+ .build();
+
+ public static final PropertyDescriptor RECORD_METADATA_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Record Metadata Strategy")
+ .description("Specifies whether the Record's metadata (topic and partition) should come from the Record's metadata field or if it should come from the configured " +
+ "Topic Name and Partition / Partitioner class properties")
+ .required(true)
+ .defaultValue(RecordMetadataStrategy.FROM_PROPERTIES)
+ .allowableValues(RecordMetadataStrategy.class)
+ .dependsOn(PUBLISH_STRATEGY, PublishStrategy.USE_WRAPPER)
+ .build();
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("FlowFiles for which all content was sent to Kafka.")
+ .build();
+
+ public static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Any FlowFile that cannot be sent to Kafka will be routed to this Relationship")
+ .build();
+
+ private static final List<PropertyDescriptor> DESCRIPTORS = List.of(
+ CONNECTION_SERVICE,
+ TOPIC_NAME,
+ FAILURE_STRATEGY,
+ DELIVERY_GUARANTEE,
+ COMPRESSION_CODEC,
+ MAX_REQUEST_SIZE,
+ TRANSACTIONS_ENABLED,
+ TRANSACTIONAL_ID_PREFIX,
+ PARTITION_CLASS,
+ PARTITION,
+ MESSAGE_DEMARCATOR,
+ RECORD_READER,
+ RECORD_WRITER,
+ PUBLISH_STRATEGY,
+ MESSAGE_KEY_FIELD,
+ ATTRIBUTE_HEADER_PATTERN,
+ HEADER_ENCODING,
+ KAFKA_KEY,
+ KEY_ATTRIBUTE_ENCODING,
+ RECORD_KEY_WRITER,
+ RECORD_METADATA_STRATEGY
+ );
+
+ private static final Set<Relationship> RELATIONSHIPS = Set.of(REL_SUCCESS, REL_FAILURE);
+
+ @Override
+ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return DESCRIPTORS;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verify(final ProcessContext context, final ComponentLog verificationLogger,
+ final Map<String, String> attributes) {
+ final List<ConfigVerificationResult> verificationResults = new ArrayList<>();
+
+ final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
+
+ final boolean transactionsEnabled = context.getProperty(TRANSACTIONS_ENABLED).asBoolean();
+ final String transactionalIdPrefix = context.getProperty(TRANSACTIONAL_ID_PREFIX).evaluateAttributeExpressions().getValue();
+ final Supplier<String> transactionalIdSupplier = new TransactionIdSupplier(transactionalIdPrefix);
+ final String deliveryGuarantee = context.getProperty(DELIVERY_GUARANTEE).getValue();
+ final String compressionCodec = context.getProperty(COMPRESSION_CODEC).getValue();
+ final String partitionClass = context.getProperty(PARTITION_CLASS).getValue();
+ final ProducerConfiguration producerConfiguration = new ProducerConfiguration(
+ transactionsEnabled, transactionalIdSupplier.get(), deliveryGuarantee, compressionCodec, partitionClass);
+ final KafkaProducerService producerService = connectionService.getProducerService(producerConfiguration);
+
+ final ConfigVerificationResult.Builder verificationPartitions = new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Topic Partitions");
+
+ final String topicName = context.getProperty(TOPIC_NAME).evaluateAttributeExpressions(attributes).getValue();
+ try {
+ final List<PartitionState> partitionStates = producerService.getPartitionStates(topicName);
+ verificationPartitions
+ .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .explanation(String.format("Partitions [%d] found for Topic [%s]", partitionStates.size(), topicName));
+ } catch (final Exception e) {
+ getLogger().error("Topic [%s] Partition verification failed", topicName, e);
+ verificationPartitions
+ .outcome(ConfigVerificationResult.Outcome.FAILED)
+ .explanation(String.format("Topic [%s] Partition access failed: %s", topicName, e));
+ }
+ verificationResults.add(verificationPartitions.build());
+
+ return verificationResults;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ final List<FlowFile> flowFiles = PublishKafkaUtil.pollFlowFiles(session);
+ if (flowFiles.isEmpty()) {
+ return;
+ }
+
+ final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
+
+ final boolean transactionsEnabled = context.getProperty(TRANSACTIONS_ENABLED).asBoolean();
+ final String transactionalIdPrefix = context.getProperty(TRANSACTIONAL_ID_PREFIX).evaluateAttributeExpressions().getValue();
+ final String deliveryGuarantee = context.getProperty(DELIVERY_GUARANTEE).getValue();
+ final String compressionCodec = context.getProperty(COMPRESSION_CODEC).getValue();
+ final String partitionClass = context.getProperty(PARTITION_CLASS).getValue();
+ final ProducerConfiguration producerConfiguration = new ProducerConfiguration(
+ transactionsEnabled, transactionalIdPrefix, deliveryGuarantee, compressionCodec, partitionClass);
+
+ try (final KafkaProducerService producerService = connectionService.getProducerService(producerConfiguration)) {
+ publishFlowFiles(context, session, flowFiles, producerService);
+ } catch (final Throwable e) {
+ getLogger().error("Publishing FlowFiles failed", e);
+ context.yield();
+ }
+ }
+
+ private void publishFlowFiles(final ProcessContext context, final ProcessSession session,
+ final List<FlowFile> flowFiles, final KafkaProducerService producerService) {
+ producerService.init();
+ for (final FlowFile flowFile : flowFiles) {
+ publishFlowFile(context, session, flowFile, producerService);
+ }
+ final RecordSummary recordSummary = producerService.complete();
+ if (recordSummary.isFailure()) {
+ routeFailureStrategy(context, session, flowFiles);
+ } else {
+ routeResults(session, recordSummary.getFlowFileResults());
+ }
+ }
+
+ private void routeFailureStrategy(final ProcessContext context, final ProcessSession session, final List<FlowFile> flowFiles) {
+ final String strategy = context.getProperty(FAILURE_STRATEGY).getValue();
+ if (FailureStrategy.ROLLBACK.getValue().equals(strategy)) {
+ session.rollback();
+ context.yield();
+ } else {
+ session.transfer(flowFiles, REL_FAILURE);
+ }
+ }
+
+ private void routeResults(final ProcessSession session, final List<FlowFileResult> flowFileResults) {
+ for (final FlowFileResult flowFileResult : flowFileResults) {
+ final long msgCount = flowFileResult.getSentCount();
+ final FlowFile flowFile = session.putAttribute(flowFileResult.getFlowFile(), MSG_COUNT, String.valueOf(msgCount));
+ session.adjustCounter("Messages Sent", msgCount, true);
+ final Relationship relationship = flowFileResult.getExceptions().isEmpty() ? REL_SUCCESS : REL_FAILURE;
+ session.transfer(flowFile, relationship);
+ }
+ }
+
+ private void publishFlowFile(final ProcessContext context, final ProcessSession session,
+ final FlowFile flowFile, final KafkaProducerService producerService) {
+ final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+ final RecordSetWriterFactory keyWriterFactory = context.getProperty(RECORD_KEY_WRITER).asControllerService(RecordSetWriterFactory.class);
+
+ final String topic = context.getProperty(TOPIC_NAME).evaluateAttributeExpressions(flowFile.getAttributes()).getValue();
+ final Integer partition = getPartition(context, flowFile);
+ final PublishContext publishContext = new PublishContext(topic, partition, null, flowFile);
+
+ final PropertyValue propertyDemarcator = context.getProperty(MESSAGE_DEMARCATOR);
+ final int maxMessageSize = context.getProperty(MAX_REQUEST_SIZE).asDataSize(DataUnit.B).intValue();
+
+ final PublishStrategy publishStrategy = PublishStrategy.valueOf(context.getProperty(PUBLISH_STRATEGY).getValue());
+ final RecordMetadataStrategy metadataStrategy = RecordMetadataStrategy.valueOf(context.getProperty(RECORD_METADATA_STRATEGY).getValue());
+
+ final String kafkaKeyAttribute = context.getProperty(KAFKA_KEY).getValue();
+ final String keyAttributeEncoding = context.getProperty(KEY_ATTRIBUTE_ENCODING).getValue();
+ final String messageKeyField = context.getProperty(MESSAGE_KEY_FIELD).evaluateAttributeExpressions(flowFile).getValue();
+ final KeyFactory keyFactory = ((PublishStrategy.USE_VALUE == publishStrategy) && (messageKeyField != null))
+ ? new MessageKeyFactory(flowFile, messageKeyField, keyWriterFactory, getLogger())
+ : new AttributeKeyFactory(kafkaKeyAttribute, keyAttributeEncoding);
+
+ final String attributeHeaderPatternProperty = context.getProperty(ATTRIBUTE_HEADER_PATTERN).getValue();
+ final Pattern attributeHeaderPattern = (attributeHeaderPatternProperty == null) ? null : Pattern.compile(attributeHeaderPatternProperty);
+ final String headerEncoding = context.getProperty(HEADER_ENCODING).evaluateAttributeExpressions().getValue();
+ final Charset headerEncodingCharacterSet = Charset.forName(headerEncoding);
+ final HeadersFactory headersFactory = new AttributesHeadersFactory(attributeHeaderPattern, headerEncodingCharacterSet);
+
+ final KafkaRecordConverter kafkaRecordConverter = getKafkaRecordConverter(
+ publishStrategy, metadataStrategy, readerFactory, writerFactory, keyWriterFactory,
+ keyFactory, headersFactory, propertyDemarcator, flowFile, maxMessageSize);
+ final PublishCallback callback = new PublishCallback(
+ producerService, publishContext, kafkaRecordConverter, flowFile.getAttributes(), flowFile.getSize());
+ session.read(flowFile, callback);
+ }
+
+ private Integer getPartition(final ProcessContext context, final FlowFile flowFile) {
+ final String partitionClass = context.getProperty(PARTITION_CLASS).getValue();
+ if (PartitionStrategy.EXPRESSION_LANGUAGE_PARTITIONING.getValue().equals(partitionClass)) {
+ final String partition = context.getProperty(PARTITION).evaluateAttributeExpressions(flowFile).getValue();
+ return Objects.hashCode(partition);
+ }
+ return null;
+ }
+
+ private KafkaRecordConverter getKafkaRecordConverter(
+ final PublishStrategy publishStrategy,
+ final RecordMetadataStrategy metadataStrategy,
+ final RecordReaderFactory readerFactory,
+ final RecordSetWriterFactory writerFactory,
+ final RecordSetWriterFactory keyWriterFactory,
+ final KeyFactory keyFactory,
+ final HeadersFactory headersFactory,
+ final PropertyValue propertyValueDemarcator,
+ final FlowFile flowFile,
+ final int maxMessageSize
+ ) {
+ final KafkaRecordConverter kafkaRecordConverter;
+ if ((readerFactory != null) && (writerFactory != null)) {
+ if (publishStrategy == PublishStrategy.USE_WRAPPER) {
+ kafkaRecordConverter = new RecordWrapperStreamKafkaRecordConverter(flowFile, metadataStrategy,
+ readerFactory, writerFactory, keyWriterFactory, maxMessageSize, getLogger());
+ } else {
+ kafkaRecordConverter = new RecordStreamKafkaRecordConverter(
+ readerFactory, writerFactory, headersFactory, keyFactory, maxMessageSize, getLogger());
+ }
+ } else if (propertyValueDemarcator.isSet()) {
+ final String demarcator = propertyValueDemarcator.evaluateAttributeExpressions(flowFile).getValue();
+ kafkaRecordConverter = new DelimitedStreamKafkaRecordConverter(
+ demarcator.getBytes(StandardCharsets.UTF_8), maxMessageSize, headersFactory);
+ } else {
+ kafkaRecordConverter = new FlowFileStreamKafkaRecordConverter(maxMessageSize, headersFactory);
+ }
+ return kafkaRecordConverter;
+ }
+
+ private static class PublishCallback implements InputStreamCallback {
+ private final KafkaProducerService producerService;
+ private final PublishContext publishContext;
+ private final KafkaRecordConverter kafkaConverter;
+ private final Map<String, String> attributes;
+ private final long inputLength;
+
+ public PublishCallback(
+ final KafkaProducerService producerService,
+ final PublishContext publishContext,
+ final KafkaRecordConverter kafkaConverter,
+ final Map<String, String> attributes,
+ final long inputLength) {
+ this.producerService = producerService;
+ this.publishContext = publishContext;
+ this.kafkaConverter = kafkaConverter;
+ this.attributes = attributes;
+ this.inputLength = inputLength;
+ }
+
+ @Override
+ public void process(final InputStream in) {
+ try (final InputStream is = new BufferedInputStream(in)) {
+ final Iterator<KafkaRecord> records = kafkaConverter.convert(attributes, is, inputLength);
+ producerService.send(records, publishContext);
+ } catch (final Exception e) {
+ publishContext.setException(e); // on data pre-process failure, indicate this to controller service
+ producerService.send(Collections.emptyIterator(), publishContext);
+ }
+ }
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/common/KafkaUtils.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/common/KafkaUtils.java
new file mode 100644
index 0000000000..d5e008ff29
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/common/KafkaUtils.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.common;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class KafkaUtils {
+
+ public static Collection<String> toTopicList(final String topics) {
+ final List<String> topicList = new ArrayList<>();
+ for (final String topic : topics.split(",", 100)) {
+ final String trimmedName = topic.trim();
+ if (!trimmedName.isEmpty()) {
+ topicList.add(trimmedName);
+ }
+ }
+ return topicList;
+ }
+
+ public static List<RecordHeader> toHeadersFiltered(final ByteRecord consumerRecord,
+ final Pattern headerNamePattern) {
+ if (headerNamePattern != null) {
+ return consumerRecord.getHeaders().stream()
+ .filter(h -> headerNamePattern.matcher(h.key()).matches())
+ .collect(Collectors.toList());
+ } else {
+ return consumerRecord.getHeaders();
+ }
+ }
+
+ public static String toKeyString(final byte[] key, final KeyEncoding keyEncoding) {
+ final String keyAttributeValue;
+ if (key == null) {
+ keyAttributeValue = null;
+ } else if (KeyEncoding.HEX.equals(keyEncoding)) {
+ keyAttributeValue = Hex.encodeHexString(key);
+ } else if (KeyEncoding.UTF8.equals(keyEncoding)) {
+ keyAttributeValue = new String(key, StandardCharsets.UTF_8);
+ } else {
+ keyAttributeValue = null;
+ }
+ return keyAttributeValue;
+ }
+
+ public static Map<String, String> toAttributes(final ByteRecord consumerRecord, final KeyEncoding keyEncoding,
+ final Pattern headerNamePattern, final Charset headerEncoding,
+ final boolean commitOffsets) {
+ final Map<String, String> attributes = new LinkedHashMap<>();
+ attributes.put(KafkaFlowFileAttribute.KAFKA_TOPIC, consumerRecord.getTopic());
+ attributes.put(KafkaFlowFileAttribute.KAFKA_PARTITION, Long.toString(consumerRecord.getPartition()));
+ attributes.put(KafkaFlowFileAttribute.KAFKA_OFFSET, Long.toString(consumerRecord.getOffset()));
+ attributes.put(KafkaFlowFileAttribute.KAFKA_CONSUMER_OFFSETS_COMMITTED, String.valueOf(commitOffsets));
+ consumerRecord.getHeaders().stream()
+ .filter(h -> h.key().equals(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET)).findFirst()
+ .ifPresent(h -> attributes.put(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET, new String(h.value(), headerEncoding)));
+ consumerRecord.getHeaders().stream()
+ .filter(h -> h.key().equals(KafkaFlowFileAttribute.KAFKA_COUNT)).findFirst()
+ .ifPresent(h -> attributes.put(KafkaFlowFileAttribute.KAFKA_COUNT, new String(h.value(), headerEncoding)));
+ attributes.put(KafkaFlowFileAttribute.KAFKA_TIMESTAMP, Long.toString(consumerRecord.getTimestamp()));
+ Optional.ofNullable(toKeyString(consumerRecord.getKey().orElse(null), keyEncoding))
+ .ifPresent(keyAttribute -> attributes.put(KafkaFlowFileAttribute.KAFKA_KEY, keyAttribute));
+
+ final List<RecordHeader> headers = consumerRecord.getHeaders();
+ attributes.put(KafkaFlowFileAttribute.KAFKA_HEADER_COUNT, Integer.toString(headers.size()));
+
+ if (headerNamePattern != null) {
+ for (final RecordHeader header : headers) {
+ final String name = header.key();
+ if (headerNamePattern.matcher(name).matches()) {
+ final String value = new String(header.value(), headerEncoding);
+ attributes.put(name, value);
+ }
+ }
+ }
+
+ return attributes;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/OffsetTracker.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/OffsetTracker.java
new file mode 100644
index 0000000000..626cf956db
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/OffsetTracker.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer;
+
+import org.apache.nifi.kafka.service.api.common.OffsetSummary;
+import org.apache.nifi.kafka.service.api.common.TopicPartitionSummary;
+import org.apache.nifi.kafka.service.api.consumer.PollingContext;
+import org.apache.nifi.kafka.service.api.consumer.PollingSummary;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class OffsetTracker {
+ private final Map<TopicPartitionSummary, OffsetSummary> offsets;
+
+ public OffsetTracker() {
+ offsets = new LinkedHashMap<>();
+ }
+
+ public void update(final ByteRecord consumerRecord) {
+ final TopicPartitionSummary topicPartitionSummary = new TopicPartitionSummary(consumerRecord.getTopic(), consumerRecord.getPartition());
+ final long offset = consumerRecord.getOffset();
+ final OffsetSummary offsetSummary = offsets.computeIfAbsent(topicPartitionSummary, (summary) -> new OffsetSummary(offset));
+ offsetSummary.setOffset(offset);
+ }
+
+ public PollingSummary getPollingSummary(final PollingContext pollingContext) {
+ final PollingSummary pollingSummary;
+ if (pollingContext.getTopicPattern().isPresent()) {
+ pollingSummary = new PollingSummary(pollingContext.getGroupId(), pollingContext.getTopicPattern().get(),
+ pollingContext.getAutoOffsetReset(), pollingContext.getMaxUncommittedTime(), offsets);
+ } else {
+ pollingSummary = new PollingSummary(pollingContext.getGroupId(), pollingContext.getTopics(),
+ pollingContext.getAutoOffsetReset(), pollingContext.getMaxUncommittedTime(), offsets);
+ }
+ return pollingSummary;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/ProcessingStrategy.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/ProcessingStrategy.java
new file mode 100644
index 0000000000..a288b58344
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/ProcessingStrategy.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer;
+
+import org.apache.nifi.components.DescribedValue;
+
+/**
+ * Enumeration of supporting strategies for consuming and serializing Kafka Records
+ */
+public enum ProcessingStrategy implements DescribedValue {
+ FLOW_FILE("Write one FlowFile for each Kafka Record consumed"),
+
+ DEMARCATOR("Write one FlowFile for each batch of Kafka Records consumed (optionally grouped by Kafka key)"),
+
+ RECORD("Write one FlowFile containing multiple Kafka Records consumed and processed with Record Reader and Record Writer");
+
+ private final String description;
+
+ ProcessingStrategy(final String description) {
+ this.description = description;
+ }
+
+ @Override
+ public String getValue() {
+ return name();
+ }
+
+ @Override
+ public String getDisplayName() {
+ return name();
+ }
+
+ @Override
+ public String getDescription() {
+ return description;
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/BundleKey.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/BundleKey.java
new file mode 100644
index 0000000000..d8cdf8ee5b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/BundleKey.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.bundle;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.nifi.kafka.service.api.common.TopicPartitionSummary;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class BundleKey {
+ final TopicPartitionSummary topicPartition;
+ final long timestamp;
+ final List<RecordHeader> headers;
+ final List<RecordHeader> headersFiltered;
+ final Map<String, String> attributes;
+ final byte[] messageKey;
+
+ public BundleKey(final TopicPartitionSummary topicPartition,
+ final long timestamp,
+ final List<RecordHeader> headers,
+ final List<RecordHeader> headersFiltered,
+ final Map<String, String> attributes,
+ final byte[] messageKey) {
+ this.topicPartition = Objects.requireNonNull(topicPartition);
+ this.timestamp = timestamp;
+ this.headers = Objects.requireNonNull(headers);
+ this.headersFiltered = Objects.requireNonNull(headersFiltered);
+ this.attributes = attributes;
+ this.messageKey = messageKey;
+ }
+
+ public TopicPartitionSummary getTopicPartition() {
+ return topicPartition;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public List<RecordHeader> getHeaders() {
+ return headers;
+ }
+
+ public Map<String, String> getAttributes() {
+ return attributes;
+ }
+
+ public byte[] getMessageKey() {
+ return messageKey;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final BundleKey bundleKey = (BundleKey) o;
+
+ return new EqualsBuilder()
+ .append(topicPartition, bundleKey.topicPartition)
+ .append(headersFiltered, bundleKey.headersFiltered)
+ .append(messageKey, bundleKey.messageKey)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 37)
+ .append(topicPartition)
+ .append(headersFiltered)
+ .append(messageKey)
+ .toHashCode();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/BundleValue.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/BundleValue.java
new file mode 100644
index 0000000000..2b92686056
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/BundleValue.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.bundle;
+
+import java.io.ByteArrayOutputStream;
+
+public class BundleValue {
+ private final ByteArrayOutputStream bos;
+ private final long firstOffset;
+ private long lastOffset;
+ private long count;
+
+ public BundleValue(final long offset) {
+ this.bos = new ByteArrayOutputStream();
+ firstOffset = offset;
+ lastOffset = offset;
+ count = 0;
+ }
+
+ public void update(final byte[] demarcator, final byte[] data, final long offset) {
+ if (bos.size() > 0) {
+ bos.writeBytes(demarcator);
+ }
+ bos.writeBytes(data);
+ lastOffset = offset;
+ ++count;
+ }
+
+ public long getFirstOffset() {
+ return firstOffset;
+ }
+
+ public long getLastOffset() {
+ return lastOffset;
+ }
+
+ public long getCount() {
+ return count;
+ }
+
+ public byte[] getData() {
+ return bos.toByteArray();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/ByteRecordBundler.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/ByteRecordBundler.java
new file mode 100644
index 0000000000..bb6758b2bc
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/bundle/ByteRecordBundler.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.bundle;
+
+import org.apache.nifi.kafka.processors.common.KafkaUtils;
+import org.apache.nifi.kafka.service.api.common.TopicPartitionSummary;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.shared.attribute.KafkaFlowFileAttribute;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public class ByteRecordBundler {
+ private final byte[] demarcator;
+ private final boolean separateByKey;
+ private final KeyEncoding keyEncoding;
+ private final Pattern headerNamePattern;
+ private final Charset headerEncoding;
+ private final boolean commitOffsets;
+
+ private final Map<BundleKey, BundleValue> bundles;
+
+ public ByteRecordBundler(final byte[] demarcator, final boolean separateByKey, final KeyEncoding keyEncoding,
+ final Pattern headerNamePattern, final Charset headerEncoding, final boolean commitOffsets) {
+ this.demarcator = demarcator;
+ this.separateByKey = separateByKey;
+ this.keyEncoding = keyEncoding;
+ this.headerNamePattern = headerNamePattern;
+ this.headerEncoding = headerEncoding;
+ this.commitOffsets = commitOffsets;
+ this.bundles = new HashMap<>();
+ }
+
+ public Iterator<ByteRecord> bundle(final Iterator<ByteRecord> consumerRecords) {
+ while (consumerRecords.hasNext()) {
+ update(bundles, consumerRecords.next());
+ }
+ return bundles.entrySet().stream()
+ .map(e -> toByteRecord(e.getKey(), e.getValue())).iterator();
+ }
+
+ private ByteRecord toByteRecord(final BundleKey key, final BundleValue value) {
+ final TopicPartitionSummary topicPartition = key.getTopicPartition();
+ key.headers.add(new RecordHeader(KafkaFlowFileAttribute.KAFKA_MAX_OFFSET,
+ Long.toString(value.getLastOffset()).getBytes(StandardCharsets.UTF_8)));
+ key.headers.add(new RecordHeader(KafkaFlowFileAttribute.KAFKA_COUNT,
+ Long.toString(value.getCount()).getBytes(StandardCharsets.UTF_8)));
+ return new ByteRecord(topicPartition.getTopic(), topicPartition.getPartition(),
+ value.getFirstOffset(), key.getTimestamp(), key.getHeaders(), key.getMessageKey(), value.getData());
+ }
+
+ private void update(final Map<BundleKey, BundleValue> bundles, final ByteRecord byteRecord) {
+ final TopicPartitionSummary topicPartition = new TopicPartitionSummary(byteRecord.getTopic(), byteRecord.getPartition());
+ final List<RecordHeader> headers = byteRecord.getHeaders();
+ final List<RecordHeader> headersFiltered = KafkaUtils.toHeadersFiltered(byteRecord, headerNamePattern);
+ final byte[] messageKey = (separateByKey ? byteRecord.getKey().orElse(null) : null);
+ final Map<String, String> attributes = KafkaUtils.toAttributes(byteRecord, keyEncoding, headerNamePattern, headerEncoding, commitOffsets);
+ final BundleKey bundleKey = new BundleKey(topicPartition, byteRecord.getTimestamp(), headers, headersFiltered, attributes, messageKey);
+ if (bundles.containsKey(bundleKey)) {
+ update(bundles, byteRecord, bundleKey);
+ } else {
+ create(bundles, byteRecord, bundleKey);
+ }
+ }
+
+ private void update(final Map<BundleKey, BundleValue> bundles, final ByteRecord byteRecord, final BundleKey bundleKey) {
+ final BundleValue bundleValue = bundles.get(bundleKey);
+ bundleValue.update(demarcator, byteRecord.getValue(), byteRecord.getOffset());
+ }
+
+ private void create(final Map<BundleKey, BundleValue> bundles, final ByteRecord byteRecord, final BundleKey bundleKey) {
+ final BundleValue bundleValue = new BundleValue(byteRecord.getOffset());
+ bundleValue.update(demarcator, byteRecord.getValue(), byteRecord.getOffset());
+ bundles.put(bundleKey, bundleValue);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/FlowFileStreamKafkaMessageConverter.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/FlowFileStreamKafkaMessageConverter.java
new file mode 100644
index 0000000000..204e85627b
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/FlowFileStreamKafkaMessageConverter.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.convert;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.kafka.processors.ConsumeKafka;
+import org.apache.nifi.kafka.processors.common.KafkaUtils;
+import org.apache.nifi.kafka.processors.consumer.OffsetTracker;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.provenance.ProvenanceReporter;
+
+import java.nio.charset.Charset;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public class FlowFileStreamKafkaMessageConverter implements KafkaMessageConverter {
+ private final Charset headerEncoding;
+ private final Pattern headerNamePattern;
+ private final KeyEncoding keyEncoding;
+ private final boolean commitOffsets;
+ private final OffsetTracker offsetTracker;
+ private final Runnable onSuccess;
+
+ public FlowFileStreamKafkaMessageConverter(
+ final Charset headerEncoding,
+ final Pattern headerNamePattern,
+ final KeyEncoding keyEncoding,
+ final boolean commitOffsets,
+ final OffsetTracker offsetTracker,
+ final Runnable onSuccess) {
+ this.headerEncoding = headerEncoding;
+ this.headerNamePattern = headerNamePattern;
+ this.keyEncoding = keyEncoding;
+ this.commitOffsets = commitOffsets;
+ this.offsetTracker = offsetTracker;
+ this.onSuccess = onSuccess;
+ }
+
+ @Override
+ public void toFlowFiles(final ProcessSession session, final Iterator<ByteRecord> consumerRecords) {
+ while (consumerRecords.hasNext()) {
+ final ByteRecord consumerRecord = consumerRecords.next();
+
+ final byte[] value = consumerRecord.getValue();
+ FlowFile flowFile = session.create();
+ flowFile = session.write(flowFile, outputStream -> outputStream.write(value));
+
+ final Map<String, String> attributes = KafkaUtils.toAttributes(
+ consumerRecord, keyEncoding, headerNamePattern, headerEncoding, commitOffsets);
+ flowFile = session.putAllAttributes(flowFile, attributes);
+
+ final ProvenanceReporter provenanceReporter = session.getProvenanceReporter();
+ final String transitUri = String.format(TRANSIT_URI_FORMAT, consumerRecord.getTopic(), consumerRecord.getPartition());
+ provenanceReporter.receive(flowFile, transitUri);
+
+ session.transfer(flowFile, ConsumeKafka.SUCCESS);
+ offsetTracker.update(consumerRecord);
+ }
+ onSuccess.run();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/KafkaMessageConverter.java
similarity index 54%
copy from nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
copy to nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/KafkaMessageConverter.java
index 15aa16ed76..113771f7f3 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-shared/src/main/java/org/apache/nifi/kafka/shared/attribute/KafkaFlowFileAttribute.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/KafkaMessageConverter.java
@@ -14,31 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.kafka.shared.attribute;
+package org.apache.nifi.kafka.processors.consumer.convert;
-/**
- * Kafka FlowFile Attributes
- */
-public interface KafkaFlowFileAttribute {
- String KAFKA_KEY = "kafka.key";
-
- String KAFKA_TOPIC = "kafka.topic";
-
- String KAFKA_PARTITION = "kafka.partition";
-
- String KAFKA_OFFSET = "kafka.offset";
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.processor.ProcessSession;
- String KAFKA_MAX_OFFSET = "kafka.max.offset";
+import java.util.Iterator;
- String KAFKA_LEADER_EPOCH = "kafka.leader.epoch";
-
- String KAFKA_TIMESTAMP = "kafka.timestamp";
-
- String KAFKA_COUNT = "kafka.count";
-
- String KAFKA_CONSUMER_GROUP_ID = "kafka.consumer.id";
+/**
+ * Implementations of {@link KafkaMessageConverter} employ specialized strategies for converting Kafka messages into
+ * NiFi FlowFiles.
+ */
+public interface KafkaMessageConverter {
- String KAFKA_CONSUMER_OFFSETS_COMMITTED = "kafka.consumer.offsets.committed";
+ void toFlowFiles(final ProcessSession session, final Iterator<ByteRecord> consumerRecords);
- String KAFKA_TOMBSTONE = "kafka.tombstone";
+ String TRANSIT_URI_FORMAT = "kafka://%s/%s";
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverter.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverter.java
new file mode 100644
index 0000000000..306cb6b149
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverter.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.convert;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.kafka.processors.ConsumeKafka;
+import org.apache.nifi.kafka.processors.common.KafkaUtils;
+import org.apache.nifi.kafka.processors.consumer.OffsetTracker;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.provenance.ProvenanceReporter;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public class RecordStreamKafkaMessageConverter implements KafkaMessageConverter {
+ private final RecordReaderFactory readerFactory;
+ private final RecordSetWriterFactory writerFactory;
+ private final Charset headerEncoding;
+ private final Pattern headerNamePattern;
+ private final KeyEncoding keyEncoding;
+ private final boolean commitOffsets;
+ private final OffsetTracker offsetTracker;
+ private final Runnable onSuccess;
+ private final ComponentLog logger;
+
+ public RecordStreamKafkaMessageConverter(
+ final RecordReaderFactory readerFactory,
+ final RecordSetWriterFactory writerFactory,
+ final Charset headerEncoding,
+ final Pattern headerNamePattern,
+ final KeyEncoding keyEncoding,
+ final boolean commitOffsets,
+ final OffsetTracker offsetTracker,
+ final Runnable onSuccess,
+ final ComponentLog logger) {
+ this.readerFactory = readerFactory;
+ this.writerFactory = writerFactory;
+ this.headerEncoding = headerEncoding;
+ this.headerNamePattern = headerNamePattern;
+ this.keyEncoding = keyEncoding;
+ this.commitOffsets = commitOffsets;
+ this.offsetTracker = offsetTracker;
+ this.onSuccess = onSuccess;
+ this.logger = logger;
+ }
+
+ @Override
+ public void toFlowFiles(final ProcessSession session,
+ final Iterator<ByteRecord> consumerRecords) {
+ try {
+ while (consumerRecords.hasNext()) {
+ final ByteRecord consumerRecord = consumerRecords.next();
+ final byte[] valueIn = consumerRecord.getValue();
+ if (valueIn.length > 0) {
+ final InputStream in = new ByteArrayInputStream(valueIn);
+ final Map<String, String> attributes = KafkaUtils.toAttributes(
+ consumerRecord, keyEncoding, headerNamePattern, headerEncoding, commitOffsets);
+ final RecordReader reader = readerFactory.createRecordReader(attributes, in, valueIn.length, logger);
+ toFlowFile(session, attributes, reader, consumerRecord);
+ offsetTracker.update(consumerRecord);
+ }
+ }
+ onSuccess.run();
+ } catch (MalformedRecordException | SchemaNotFoundException | IOException e) {
+ throw new ProcessException("FlowFile Record conversion failed", e);
+ }
+ }
+
+ private void toFlowFile(final ProcessSession session,
+ final Map<String, String> attributes,
+ final RecordReader reader,
+ final ByteRecord consumerRecord) throws IOException, SchemaNotFoundException {
+ int recordCount = 0;
+ FlowFile flowFile = session.create();
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ try (final OutputStream rawOut = session.write(flowFile)) {
+ final RecordSet recordSet = reader.createRecordSet();
+ final RecordSchema schema = writerFactory.getSchema(attributes, recordSet.getSchema());
+ final RecordSetWriter writer = writerFactory.createWriter(logger, schema, rawOut, attributes);
+ Record record;
+ writer.beginRecordSet();
+ while ((record = recordSet.next()) != null) {
+ ++recordCount;
+ writer.write(record);
+ }
+ writer.finishRecordSet();
+ writer.flush();
+ final ProvenanceReporter provenanceReporter = session.getProvenanceReporter();
+ final String transitUri = String.format(TRANSIT_URI_FORMAT, consumerRecord.getTopic(), consumerRecord.getPartition());
+ provenanceReporter.receive(flowFile, transitUri);
+ }
+ flowFile = session.putAttribute(flowFile, "record.count", String.valueOf(recordCount));
+ session.transfer(flowFile, ConsumeKafka.SUCCESS);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/WrapperRecordStreamKafkaMessageConverter.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/WrapperRecordStreamKafkaMessageConverter.java
new file mode 100644
index 0000000000..e1a0d93123
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/convert/WrapperRecordStreamKafkaMessageConverter.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.convert;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.kafka.processors.ConsumeKafka;
+import org.apache.nifi.kafka.processors.common.KafkaUtils;
+import org.apache.nifi.kafka.processors.consumer.OffsetTracker;
+import org.apache.nifi.kafka.processors.consumer.wrapper.ConsumeWrapperRecord;
+import org.apache.nifi.kafka.processors.consumer.wrapper.WrapperRecordKeyReader;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.kafka.shared.property.KeyEncoding;
+import org.apache.nifi.kafka.shared.property.KeyFormat;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.provenance.ProvenanceReporter;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+import org.apache.nifi.util.Tuple;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public class WrapperRecordStreamKafkaMessageConverter implements KafkaMessageConverter {
+ private final RecordReaderFactory readerFactory;
+ private final RecordSetWriterFactory writerFactory;
+ private final RecordReaderFactory keyReaderFactory;
+ private final Charset headerEncoding;
+ private final Pattern headerNamePattern;
+ private final KeyFormat keyFormat;
+ private final KeyEncoding keyEncoding;
+ private final boolean commitOffsets;
+ private final OffsetTracker offsetTracker;
+ private final Runnable onSuccess;
+ private final ComponentLog logger;
+
+ public WrapperRecordStreamKafkaMessageConverter(
+ final RecordReaderFactory readerFactory,
+ final RecordSetWriterFactory writerFactory,
+ final RecordReaderFactory keyReaderFactory,
+ final Charset headerEncoding,
+ final Pattern headerNamePattern,
+ final KeyFormat keyFormat,
+ final KeyEncoding keyEncoding,
+ final boolean commitOffsets,
+ final OffsetTracker offsetTracker,
+ final Runnable onSuccess,
+ final ComponentLog logger) {
+ this.readerFactory = readerFactory;
+ this.writerFactory = writerFactory;
+ this.keyReaderFactory = keyReaderFactory;
+ this.headerEncoding = headerEncoding;
+ this.headerNamePattern = headerNamePattern;
+ this.keyFormat = keyFormat;
+ this.keyEncoding = keyEncoding;
+ this.commitOffsets = commitOffsets;
+ this.offsetTracker = offsetTracker;
+ this.onSuccess = onSuccess;
+ this.logger = logger;
+ }
+
+ @Override
+ public void toFlowFiles(final ProcessSession session,
+ final Iterator<ByteRecord> consumerRecords) {
+ try {
+ while (consumerRecords.hasNext()) {
+ final ByteRecord consumerRecord = consumerRecords.next();
+ final byte[] value = consumerRecord.getValue();
+ if (value.length > 0) {
+ final InputStream in = new ByteArrayInputStream(value);
+ final Map<String, String> attributes = KafkaUtils.toAttributes(
+ consumerRecord, keyEncoding, headerNamePattern, headerEncoding, commitOffsets);
+
+ final WrapperRecordKeyReader keyReader = new WrapperRecordKeyReader(
+ keyFormat, keyReaderFactory, keyEncoding, logger);
+ final Tuple<RecordField, Object> recordKey = keyReader.toWrapperRecordKey(
+ consumerRecord.getKey().orElse(null), attributes);
+
+ final RecordReader reader = readerFactory.createRecordReader(attributes, in, value.length, logger);
+ toFlowFile(session, attributes, reader, consumerRecord, recordKey);
+ offsetTracker.update(consumerRecord);
+ }
+ }
+ onSuccess.run();
+ } catch (MalformedRecordException | SchemaNotFoundException | IOException e) {
+ throw new ProcessException("FlowFile Record conversion failed", e);
+ }
+ }
+
+ private void toFlowFile(final ProcessSession session,
+ final Map<String, String> attributes,
+ final RecordReader reader,
+ final ByteRecord consumerRecord,
+ Tuple<RecordField, Object> recordKey)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int recordCount = 0;
+ FlowFile flowFile = session.create();
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ try (final OutputStream rawOut = session.write(flowFile)) {
+ final RecordSet recordSet = reader.createRecordSet();
+ final RecordSchema schema = writerFactory.getSchema(attributes, recordSet.getSchema());
+ final RecordSchema schemaWrapper = WrapperRecord.toWrapperSchema(recordKey.getKey(), schema);
+ final RecordSetWriter writer = writerFactory.createWriter(logger, schemaWrapper, rawOut, attributes);
+ Record record;
+ writer.beginRecordSet();
+ while ((record = recordSet.next()) != null) {
+ ++recordCount;
+ final ConsumeWrapperRecord consumeWrapperRecord = new ConsumeWrapperRecord(headerEncoding);
+ final MapRecord wrapperRecord = consumeWrapperRecord.toWrapperRecord(consumerRecord, record, recordKey);
+ writer.write(wrapperRecord);
+ }
+ writer.finishRecordSet();
+ writer.flush();
+ final ProvenanceReporter provenanceReporter = session.getProvenanceReporter();
+ final String transitUri = String.format(TRANSIT_URI_FORMAT, consumerRecord.getTopic(), consumerRecord.getPartition());
+ provenanceReporter.receive(flowFile, transitUri);
+ }
+ flowFile = session.putAttribute(flowFile, "record.count", String.valueOf(recordCount));
+ session.transfer(flowFile, ConsumeKafka.SUCCESS);
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/wrapper/ConsumeWrapperRecord.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/wrapper/ConsumeWrapperRecord.java
new file mode 100644
index 0000000000..e1f2380044
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/consumer/wrapper/ConsumeWrapperRecord.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors.consumer.wrapper;
+
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
... 3602 lines suppressed ...