You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2025/12/09 01:04:22 UTC
(nifi) branch NIFI-15311 created (now 7d2ef18208)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a change to branch NIFI-15311
in repository https://gitbox.apache.org/repos/asf/nifi.git
at 7d2ef18208 NIFI-15258: Initial implementation of Connectors
This branch includes the following new commits:
new 7d2ef18208 NIFI-15258: Initial implementation of Connectors
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(nifi) 01/01: NIFI-15258: Initial implementation of Connectors
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch NIFI-15311
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 7d2ef182080102e69fa188a8f6b0ee99b2c955f4
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Tue Sep 2 18:22:14 2025 -0400
NIFI-15258: Initial implementation of Connectors
---
.cursorrules | 71 +
.../c2/command/DefaultFlowStateStrategyTest.java | 9 +-
nifi-commons/nifi-connector-utils/pom.xml | 42 +
.../connector/util/VersionedFlowUtils.java | 366 ++++
.../java/org/apache/nifi/util/NiFiProperties.java | 54 +-
nifi-commons/pom.xml | 1 +
.../nifi-connector-mock-api/pom.xml | 35 +
.../server/ConnectorConfigVerificationResult.java | 21 +-
.../mock/connector/server/ConnectorMockServer.java | 23 +-
.../mock/connector/server/ConnectorTestRunner.java | 51 +
.../nifi-connector-mock-server-nar/pom.xml | 55 +
.../nifi-connector-mock-server/pom.xml | 73 +
.../mock/connector/server/MockAuditService.java | 63 +
.../server/MockConnectorInitializationContext.java | 156 ++
.../connector/server/MockConnectorRepository.java | 25 +-
.../server/MockExtensionDiscoveringManager.java | 60 +
.../mock/connector/server/MockExtensionMapper.java | 44 +
.../server/MockRuleViolationsManager.java | 80 +
.../server/MockServerConfigVerificationResult.java | 52 +
.../mock/connector/server/MockStateManager.java | 89 +
.../connector/server/MockStateManagerProvider.java | 59 +
.../nifi/mock/connector/server/MockStateMap.java | 56 +
.../connector/server/NopPropertyEncryptor.java | 26 +-
.../mock/connector/server/PermitAllAuthorizer.java | 46 +
.../server/StandardConnectorMockServer.java | 304 +++
.../META-INF/services/org.apache.nifi.NiFiServer | 5 +-
...e.nifi.components.connector.ConnectorRepository | 5 +-
.../server/MockStateManagerProviderTest.java | 267 +++
.../nifi-connector-mock-integration-tests/pom.xml | 118 ++
.../mock/connectors/tests/CreateConnectorIT.java | 30 +-
.../mock/connectors/tests/MockProcessorIT.java | 91 +
.../pom.xml | 35 +
.../nifi-connector-mock-test-connectors/pom.xml | 48 +
.../nifi/mock/connectors/GenerateAndLog.java | 55 +
.../org.apache.nifi.components.connector.Connector | 5 +-
.../main/resources/flows/Generate_and_Update.json | 1 +
.../nifi-connector-mock-test-bundle/pom.xml | 33 +
.../nifi-connector-mock/pom.xml | 147 ++
.../connector/StandardConnectorTestRunner.java | 193 ++
.../src/main/resources/nifi.properties | 159 ++
nifi-connector-mock-bundle/pom.xml | 34 +
.../nifi-kafka-to-s3-connector/pom.xml | 40 +
.../connectors/kafkas3/KafkaConnectionStep.java | 135 ++
.../apache/nifi/connectors/kafkas3/KafkaToS3.java | 295 +++
.../connectors/kafkas3/KafkaToS3FlowBuilder.java | 217 ++
.../nifi/connectors/kafkas3/KafkaTopicsStep.java | 85 +
.../org/apache/nifi/connectors/kafkas3/S3Step.java | 156 ++
.../org.apache.nifi.components.connector.Connector | 5 +-
.../src/main/resources/flows/Kafka_to_S3.json | 2105 ++++++++++++++++++++
.../nifi-kafka-to-s3-integration-tests/pom.xml | 226 +++
.../nifi/connectors/kafkas3/KafkaToS3IT.java | 742 +++++++
.../nifi-kafka-to-s3-nar/pom.xml | 36 +
nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml | 39 +
nifi-connectors/pom.xml | 42 +
.../processors/aws/s3/AbstractS3Processor.java | 11 +
.../nifi/processors/aws/region/RegionUtil.java | 2 +-
.../apache/nifi/kafka/processors/ConsumeKafka.java | 118 +-
.../nifi/kafka/processors/ConsumeKafkaTest.java | 22 +-
.../kafka/service/api/consumer/PollingContext.java | 4 +-
.../kafka/service/Kafka3ConnectionService.java | 40 +-
.../service/consumer/Kafka3AssignmentService.java | 93 +
.../service/consumer/Kafka3ConsumerService.java | 57 -
.../kafka/service/consumer/RecordIterable.java | 79 +
.../processors/standard/DuplicateFlowFile.java | 26 +-
.../nifi/controller/queue/DropFlowFileStatus.java | 7 +
.../TestVolatileProvenanceRepository.java | 6 +-
.../nifi/authorization/resource/ResourceType.java | 3 +-
.../ClusteredConnectorRequestReplicator.java | 87 +
.../ConnectorRequestReplicatorConfiguration.java | 46 +
.../FrameworkClusterConfiguration.java | 11 +-
.../TestThreadPoolRequestReplicator.java | 6 +-
.../nifi/controller/StandardProcessorNode.java | 102 +-
.../nifi/controller/flow/AbstractFlowManager.java | 2 +-
.../controller/flowanalysis/FlowAnalysisUtil.java | 6 +-
.../repository/StandardProcessSession.java | 31 +-
...StandardControllerServiceInvocationHandler.java | 40 +-
.../service/StandardControllerServiceNode.java | 91 +-
.../service/StandardControllerServiceProvider.java | 4 +-
.../service/StandardControllerServiceResolver.java | 6 +-
.../nifi/events/VolatileBulletinRepository.java | 1 +
.../StandardVersionedComponentSynchronizer.java | 34 +-
.../apache/nifi/groups/StandardProcessGroup.java | 57 +-
...rLogObserver.java => ConnectorLogObserver.java} | 21 +-
.../apache/nifi/logging/ProcessorLogObserver.java | 2 +-
.../StandardParameterReferenceManager.java | 17 +-
.../nifi/processor/StandardValidationContext.java | 28 +-
.../StandardValidationContextFactory.java | 6 +-
.../flow/FlowAnalyzingRegistryClientNode.java | 13 +-
...pper.java => VersionedComponentFlowMapper.java} | 100 +-
.../nifi/remote/StandardRemoteProcessGroup.java | 13 +-
.../repository/StandardProcessSessionTest.java | 4 +
.../StandardControllerServiceResolverTest.java | 6 +-
.../flow/FlowAnalyzingRegistryClientNodeTest.java | 4 +-
....java => TestVersionedComponentFlowMapper.java} | 10 +-
.../connector/ConfigurationStepConfiguration.java} | 19 +-
.../connector/ConfigurationUpdateResult.java} | 20 +-
.../connector/ConnectorConfiguration.java | 57 +
.../nifi/components/connector/ConnectorNode.java | 211 ++
.../components/connector/ConnectorRepository.java | 95 +
.../ConnectorRepositoryInitializationContext.java} | 24 +-
.../connector/ConnectorRequestReplicator.java} | 20 +-
.../connector/ConnectorState.java} | 28 +-
.../connector/ConnectorStateTransition.java | 82 +
.../connector/FlowContextFactory.java} | 21 +-
.../FrameworkConnectorInitializationContext.java} | 18 +-
...workConnectorInitializationContextBuilder.java} | 26 +-
.../connector/FrameworkFlowContext.java} | 23 +-
.../MutableConnectorConfigurationContext.java | 57 +
.../connector/ParameterContextFacadeFactory.java} | 21 +-
.../connector/ProcessGroupFacadeFactory.java} | 22 +-
.../connector/ProcessGroupFactory.java} | 20 +-
.../DisabledServiceValidationResult.java | 14 +
.../org/apache/nifi/connectable/Connectable.java | 2 +
.../connectable/ConnectableFlowFileActivity.java | 70 +
.../FlowFileActivity.java} | 24 +-
.../nifi/connectable/FlowFileTransferCounts.java | 88 +
.../connectable/ProcessGroupFlowFileActivity.java | 149 ++
.../nifi/controller/AbstractComponentNode.java | 48 +-
.../org/apache/nifi/controller/AbstractPort.java | 10 +
.../org/apache/nifi/controller/ComponentNode.java | 3 +
.../apache/nifi/controller/ProcessScheduler.java | 15 +-
.../org/apache/nifi/controller/ProcessorNode.java | 17 +-
.../org/apache/nifi/controller/StandardFunnel.java | 9 +
.../nifi/controller/ValidationContextFactory.java | 4 +-
.../apache/nifi/controller/flow/FlowManager.java | 17 +
.../nifi/controller/flow/VersionedDataflow.java | 10 +
.../nifi/controller/queue/DropFlowFileRequest.java | 16 +
.../controller/service/ControllerServiceNode.java | 10 +-
.../service/ControllerServiceProvider.java | 4 +-
.../org/apache/nifi/events/BulletinFactory.java | 7 +
.../java/org/apache/nifi/groups/ProcessGroup.java | 26 +-
.../org/apache/nifi/groups/RemoteProcessGroup.java | 4 +-
.../nifi-framework/nifi-framework-core/pom.xml | 7 +-
.../components/connector/ConnectorDetails.java} | 33 +-
.../connector/ConnectorParameterLookup.java | 176 ++
.../ConnectorValidationContextBridge.java | 134 ++
.../connector/DescribedValueProvider.java} | 18 +-
.../nifi/components/connector/GhostConnector.java | 129 ++
.../StandaloneConnectorRequestReplicator.java} | 30 +-
.../StandardConnectorConfigurationContext.java | 328 +++
.../StandardConnectorInitializationContext.java | 122 ++
.../connector/StandardConnectorNode.java | 806 ++++++++
.../connector/StandardConnectorPropertyValue.java | 101 +
...StandardConnectorRepoInitializationContext.java | 59 +
.../connector/StandardConnectorRepository.java | 187 ++
.../StandardConnectorStateTransition.java | 113 ++
.../StandardConnectorValidationContext.java | 113 ++
.../components/connector/StandardFlowContext.java | 119 ++
.../standalone/ComponentContextProvider.java | 41 +
...egalExecutionEngineStatelessGroupLifecycle.java | 46 +
.../standalone/StandaloneConnectionFacade.java | 49 +
.../StandaloneControllerServiceFacade.java | 172 ++
.../StandaloneControllerServiceLifecycle.java | 62 +
.../StandaloneParameterContextFacade.java | 208 ++
.../standalone/StandaloneProcessGroupFacade.java | 380 ++++
.../StandaloneProcessGroupLifecycle.java | 294 +++
.../standalone/StandaloneProcessorFacade.java | 180 ++
.../standalone/StandaloneProcessorLifecycle.java | 93 +
.../StandaloneStatelessGroupLifecycle.java | 51 +
.../apache/nifi/controller/ExtensionBuilder.java | 340 ++--
.../org/apache/nifi/controller/FlowController.java | 75 +-
.../flow/FlowControllerFlowContextFactory.java | 114 ++
.../FlowControllerProcessGroupFacadeFactory.java | 70 +
.../flow/StandardComponentContextProvider.java | 105 +
.../nifi/controller/flow/StandardFlowManager.java | 154 +-
.../flow/StandardStatelessGroupNodeFactory.java | 4 +-
.../flowanalysis/StandardFlowAnalysisContext.java | 4 +-
.../scheduling/StandardProcessScheduler.java | 89 +-
.../serialization/VersionedDataflowMapper.java | 20 +-
.../serialization/VersionedFlowSynchronizer.java | 184 ++
.../VersionedReportingTaskSnapshotMapper.java | 6 +-
.../nifi/flowanalysis/StandardFlowAnalyzer.java | 10 +-
.../configuration/FlowControllerConfiguration.java | 78 +-
.../nifi/groups/StandardStatelessGroupNode.java | 9 +
....nifi.components.connector.ConnectorRepository} | 5 +-
.../components/connector/BlockingConnector.java | 127 ++
.../connector/DynamicAllowableValuesConnector.java | 137 ++
.../components/connector/DynamicFlowConnector.java | 368 ++++
.../components/connector/ParameterConnector.java | 130 ++
.../components/connector/SleepingConnector.java | 121 ++
.../connector/StandardConnectorNodeIT.java | 630 ++++++
.../TestStandardConnectorConfigurationContext.java | 299 +++
...TestStandardConnectorInitializationContext.java | 128 ++
.../connector/TestStandardConnectorNode.java | 640 ++++++
.../connector/TestStandardConnectorRepository.java | 122 ++
.../connector/processors/CreateDummyFlowFile.java | 84 +
.../connector/processors/DuplicateFlowFile.java | 128 ++
.../connector/processors/ExposeFileValues.java | 72 +
.../connector/processors/LogFlowFileContents.java | 58 +
.../connector/processors/OverwriteFlowFile.java | 79 +-
.../components/connector/processors/Sleep.java | 151 ++
.../connector/processors/TerminateFlowFile.java} | 31 +-
.../connector/services/CounterService.java} | 20 +-
.../services/impl/StandardCounterService.java} | 27 +-
.../nifi/controller/MockStateManagerProvider.java | 70 +
.../apache/nifi/controller/flow/NopConnector.java | 163 ++
.../controller/flow/TestStandardFlowManager.java | 166 ++
.../TestStandardControllerServiceProvider.java | 42 +-
.../controller/service/mock/MockProcessGroup.java | 19 +
....java => VersionedComponentFlowMapperTest.java} | 4 +-
...org.apache.nifi.components.connector.Connector} | 7 +-
.../org.apache.nifi.controller.ControllerService | 5 +-
.../services/org.apache.nifi.processor.Processor | 9 +-
.../src/test/resources/flows/choose-color.json | 76 +
.../flows/generate-and-log-with-parameter.json | 253 +++
.../flows/generate-duplicate-log-flow.json | 690 +++++++
.../nar/StandardExtensionDiscoveringManager.java | 20 +-
.../java/org/apache/nifi/nar/DummyConnector.java | 115 ++
...rdExtensionDiscoveringManagerConnectorTest.java | 49 +
...org.apache.nifi.components.connector.Connector} | 7 +-
.../apache/nifi/headless/HeadlessNiFiServer.java | 36 +-
.../main/java/org/apache/nifi/nar/NarUnpacker.java | 4 +-
.../web/server/StandardServerProviderTest.java | 8 +-
.../nifi/authorization/AuthorizableLookup.java | 6 +
.../authorization/StandardAuthorizableLookup.java | 5 +
.../apache/nifi/web/StandardNiFiServiceFacade.java | 41 +-
.../web/dao/impl/StandardControllerServiceDAO.java | 2 +-
.../nifi/web/dao/impl/StandardProcessGroupDAO.java | 4 +-
.../nifi/web/dao/impl/StandardProcessorDAO.java | 3 +-
.../nifi/web/StandardNiFiServiceFacadeTest.java | 10 +-
.../nifi/web/api/TestDataTransferResource.java | 18 +-
.../dao/impl/TestStandardParameterContextDAO.java | 2 +-
.../StandardClientRegistrationProviderTest.java | 2 +-
.../StandardRegistrationBuilderProviderTest.java | 4 +-
...dardRelyingPartyRegistrationRepositoryTest.java | 4 +-
.../x509/X509AuthenticationProviderTest.java | 2 +-
.../java/org/apache/nifi/nar/ExtensionMapping.java | 25 +-
.../stateless/bootstrap/AllowListClassLoader.java | 5 +
.../stateless/bootstrap/StatelessBootstrap.java | 2 +-
.../scheduling/StatelessProcessScheduler.java | 29 +-
.../stateless/engine/StatelessFlowManager.java | 16 +
pom.xml | 4 +-
232 files changed, 19342 insertions(+), 992 deletions(-)
diff --git a/.cursorrules b/.cursorrules
new file mode 100644
index 0000000000..3bcdd1f6cd
--- /dev/null
+++ b/.cursorrules
@@ -0,0 +1,71 @@
+AI persona
+----------
+Act as an experienced Java software engineer. When considering how to implement a task, first consider the big picture of what is being asked. Then determine which classes will need to be udpated.
+Quite often, a single request will require manipulating many different classes. Generally speaking, it is best to avoid changing established interfaces, but it is often acceptable because the public API
+that NiFi adheres to is not in this module.
+
+
+API
+---
+When it is necessary to lookup the API for reference, the API can generally be found at ../nifi-api, where .. is relative to the project's root directory.
+
+Building
+--------
+NiFi is a complex Maven codebase. Never build code (testing or otherwise) using javac. Always use `mvn` instead.
+
+Code Style
+----------
+NiFi adheres to a few code styles that are not necessarily common. Please ensure that you observe these code styles.
+1. Any variable that can be marked `final` must be marked `final`. This includes declarations of Exceptions, method arguments, local variables, member variables, etc.
+2. Short-hand is highly discouraged in names of variables, classes, methods, etc., as well as in documentation.
+3. Private / helper methods should not be placed before the first public/protected method that calls it.
+4. Unless the method is to be heavily reused, avoid creating trivial 1-2 line methods and instead just place the code inline.
+5. Code is allowed to be up to 200 characters wide. Avoid breaking lines into many short lines.
+6. Avoid creating private methods that are called only once unless they are at least 10 lines long or are complex.
+7. It is never acceptable to use star imports. Import each individual class that is to be used.
+8. Never use underscores in class names, variables, or filenames.
+9. Never use System.out.println but instead use SLF4J Loggers
+10. Avoid excessive whitespace in method invocations. For example, instead of writing:
+```
+myObject.doSomething(
+ arg1,
+ arg2,
+ arg3,
+ arg4,
+ arg5
+);
+```
+Write this instead:
+```
+myObject.doSomething(arg1, arg2, arg3, arg4, arg5);
+```
+
+11. When possible, prefer importing a class, rather than using fully qualified classname inline in the code.
+12. Avoid statically importing methods unless they are used frequently such as methods in the `Assertions` and `Mockito` classes.
+13. Avoid trailing whitespace at the end of lines, especially in blank lines.
+
+Unit / Automated Testing
+------------------------
+In addition to the general rules defined in the Code Style section, follow these rules when creating or manipulating automated tests.
+1. NEVER add comments such as `// Given`, `// When`, `// Then`. These comments are considered an anti-pattern and should be removed or replaced whenever they are encountered.
+ Instead, leave them out all together (preferred) or use comments that clearly articulate what is happening, such as `// Setup`, `// Invoke method`, `// Assert expected results.`
+ One of the reasons that this is considered an anti-pattern (in addition to the fact that the given/when/then nomeclature itself provides no meaning) is that it assumes a very specific pattern
+ in unit tests, that we will create a bunch of objects, invoke the method we care about, make assertions, and then end. This often results in many tests that are extremely repetitive.
+ Instead, whenever it makes sense to do so, create the prerequisitive objects, invoke the method we care about with appropriate arguments, make assertions, and then invoke again with a different
+ set of arguments, make assertions, etc. There is no need to have many repetitive methods that each create many repetitive objects.
+
+2. Unit tests are Java. They are not English. As such, they should be written like Java. Frameworks such as assertj that strive to make the unit tests look more "English-like" should be avoided.
+ Use of these frameworks sometimes works well but often quickly devolves into automated tests that read like neither English nor Java.
+
+3. Like any other code, unit tests should be created using reusable methods where appropriate. Do not create 15 methods that are all very similar and repetitive. Instead, create reusable methods
+ that can be called from each of the methods.
+
+4. Never use the `assert` keyword. Use JUnit assertions instead.
+
+5. Never create a test file that is centered around testing a method or capability. Unit tests must always be named after the class they are testing. It is okay if a given unit test class is very, very long.
+
+6. This is a Java project using the Maven structure. Java test files must always fall under src/test/java of the appropriate sub-module.
+
+7. Never use pointless assertions such as assertDoesNotThrow - this adds nothing but complexity. Just call the method, and if it throws an Exception, the test will fail. I.e.,
+ it is assumed by default that each line does not throw an Exception, so do not use
+
diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java
index b8dc847395..9f05795bc9 100644
--- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java
+++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java
@@ -31,7 +31,6 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
import static org.apache.nifi.c2.protocol.api.C2OperationState.OperationState.FULLY_APPLIED;
import static org.apache.nifi.c2.protocol.api.C2OperationState.OperationState.NOT_APPLIED;
@@ -111,13 +110,13 @@ public class DefaultFlowStateStrategyTest {
ProcessGroup rootProcessGroup = mock(ProcessGroup.class);
RemoteProcessGroup remoteProcessGroup = mock(RemoteProcessGroup.class);
Set<RemoteProcessGroup> remoteProcessGroups = Set.of(remoteProcessGroup);
- CompletableFuture<Void> rootProcessGroupStopFuture = mock(CompletableFuture.class);
- Future remoteProcessGroupStopFuture = mock(Future.class);
+ CompletableFuture<Void> rootProcessGroupStopFuture = CompletableFuture.completedFuture(null);
+ CompletableFuture<Void> remoteProcessGroupStopFuture = CompletableFuture.completedFuture(null);
ProcessGroup nestedProcessGroup = mock(ProcessGroup.class);
RemoteProcessGroup nestedRemoteProcessGroup = mock(RemoteProcessGroup.class);
Set<RemoteProcessGroup> nestedRemoteProcessGroups = Set.of(nestedRemoteProcessGroup);
- CompletableFuture<Void> nestedProcessGroupStopFuture = mock(CompletableFuture.class);
- Future nestedRemoteProcessGroupStopFuture = mock(Future.class);
+ CompletableFuture<Void> nestedProcessGroupStopFuture = CompletableFuture.completedFuture(null);
+ CompletableFuture<Void> nestedRemoteProcessGroupStopFuture = CompletableFuture.completedFuture(null);
when(flowManager.getRootGroup()).thenReturn(rootProcessGroup);
when(rootProcessGroup.getRemoteProcessGroups()).thenReturn(remoteProcessGroups);
diff --git a/nifi-commons/nifi-connector-utils/pom.xml b/nifi-commons/nifi-connector-utils/pom.xml
new file mode 100644
index 0000000000..2794ecbfd6
--- /dev/null
+++ b/nifi-commons/nifi-connector-utils/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-utils</artifactId>
+ <name>nifi-connector-utils</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/nifi-commons/nifi-connector-utils/src/main/java/org/apache/nifi/components/connector/util/VersionedFlowUtils.java b/nifi-commons/nifi-connector-utils/src/main/java/org/apache/nifi/components/connector/util/VersionedFlowUtils.java
new file mode 100644
index 0000000000..8eb839a3ba
--- /dev/null
+++ b/nifi-commons/nifi-connector-utils/src/main/java/org/apache/nifi/components/connector/util/VersionedFlowUtils.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.components.connector.util;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.nifi.flow.Bundle;
+import org.apache.nifi.flow.ComponentType;
+import org.apache.nifi.flow.ConnectableComponent;
+import org.apache.nifi.flow.ConnectableComponentType;
+import org.apache.nifi.flow.Position;
+import org.apache.nifi.flow.ScheduledState;
+import org.apache.nifi.flow.VersionedConnection;
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.flow.VersionedExternalFlow;
+import org.apache.nifi.flow.VersionedParameter;
+import org.apache.nifi.flow.VersionedParameterContext;
+import org.apache.nifi.flow.VersionedPort;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.flow.VersionedProcessor;
+
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Predicate;
+
+public class VersionedFlowUtils {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ public static VersionedExternalFlow loadFlowFromResource(final String resourceName) {
+ try (final InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName)) {
+ if (in == null) {
+ throw new IllegalArgumentException("Resource not found: " + resourceName);
+ }
+
+ return OBJECT_MAPPER.readValue(in, VersionedExternalFlow.class);
+ } catch (final Exception e) {
+ throw new IllegalStateException("Unable to load resource: " + resourceName, e);
+ }
+ }
+
+ public static Optional<VersionedProcessor> findProcessor(final VersionedProcessGroup group, final Predicate<VersionedProcessor> predicate) {
+ final List<VersionedProcessor> processors = findProcessors(group, predicate);
+ if (processors.size() == 1) {
+ return Optional.of(processors.getFirst());
+ }
+ return Optional.empty();
+ }
+
+ public static List<VersionedProcessor> findProcessors(final VersionedProcessGroup group, final Predicate<VersionedProcessor> predicate) {
+ final List<VersionedProcessor> processors = new ArrayList<>();
+ findProcessors(group, predicate, processors);
+ return processors;
+ }
+
+ private static void findProcessors(final VersionedProcessGroup group, final Predicate<VersionedProcessor> predicate, final List<VersionedProcessor> processors) {
+ for (final VersionedProcessor processor : group.getProcessors()) {
+ if (predicate.test(processor)) {
+ processors.add(processor);
+ }
+ }
+
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ findProcessors(childGroup, predicate, processors);
+ }
+ }
+
+ public static ConnectableComponent createConnectableComponent(final VersionedProcessor processor) {
+ final ConnectableComponent component = new ConnectableComponent();
+ component.setId(processor.getIdentifier());
+ component.setName(processor.getName());
+ component.setType(ConnectableComponentType.PROCESSOR);
+ component.setGroupId(processor.getGroupIdentifier());
+ return component;
+ }
+
+ public static ConnectableComponent createConnectableComponent(final VersionedPort port) {
+ final ConnectableComponent component = new ConnectableComponent();
+ component.setId(port.getIdentifier());
+ component.setName(port.getName());
+ component.setType(port.getComponentType() == ComponentType.INPUT_PORT ? ConnectableComponentType.INPUT_PORT : ConnectableComponentType.OUTPUT_PORT);
+ component.setGroupId(port.getGroupIdentifier());
+ return component;
+ }
+
+ public static void addConnection(final VersionedProcessGroup group, final ConnectableComponent source, final ConnectableComponent destination, final Set<String> relationships) {
+ final VersionedConnection connection = new VersionedConnection();
+ connection.setSource(source);
+ connection.setDestination(destination);
+ connection.setSelectedRelationships(relationships);
+ connection.setBends(List.of());
+ connection.setLabelIndex(0);
+ connection.setzIndex(0L);
+ connection.setGroupIdentifier(group.getIdentifier());
+ connection.setLoadBalanceStrategy("DO_NOT_LOAD_BALANCE");
+ connection.setBackPressureDataSizeThreshold("1 GB");
+ connection.setBackPressureObjectThreshold(10000L);
+ connection.setFlowFileExpiration("0 sec");
+ connection.setPrioritizers(new ArrayList<>());
+ connection.setComponentType(ComponentType.CONNECTION);
+
+ Set<VersionedConnection> connections = group.getConnections();
+ if (connections == null) {
+ connections = new HashSet<>();
+ group.setConnections(connections);
+ }
+ connections.add(connection);
+
+ final String uuid = generateDeterministicUuid(group, ComponentType.CONNECTION);
+ connection.setIdentifier(uuid);
+ }
+
+ public static List<VersionedConnection> findOutboundConnections(final VersionedProcessGroup group, final VersionedProcessor processor) {
+ final VersionedProcessGroup processorGroup = findGroupForProcessor(group, processor);
+ if (processorGroup == null) {
+ return List.of();
+ }
+
+ final List<VersionedConnection> outboundConnections = new ArrayList<>();
+ final Set<VersionedConnection> connections = processorGroup.getConnections();
+ if (connections == null) {
+ return outboundConnections;
+ }
+
+ for (final VersionedConnection connection : connections) {
+ final ConnectableComponent source = connection.getSource();
+ if (Objects.equals(source.getId(), processor.getIdentifier()) && source.getType() == ConnectableComponentType.PROCESSOR) {
+ outboundConnections.add(connection);
+ }
+ }
+
+ return outboundConnections;
+ }
+
+ public static VersionedProcessGroup findGroupForProcessor(final VersionedProcessGroup rootGroup, final VersionedProcessor processor) {
+ if (rootGroup.getProcessors().contains(processor)) {
+ return rootGroup;
+ }
+
+ for (final VersionedProcessGroup childGroup : rootGroup.getProcessGroups()) {
+ final VersionedProcessGroup foundGroup = findGroupForProcessor(childGroup, processor);
+ if (foundGroup != null) {
+ return foundGroup;
+ }
+ }
+
+ return null;
+ }
+
+ public static String generateDeterministicUuid(final VersionedProcessGroup group, final ComponentType componentType) {
+ final int componentCount = getComponentCount(group, componentType);
+ final String uuidSeed = "%s-%s-%d".formatted(group.getIdentifier(), componentType.name(), componentCount);
+ return UUID.nameUUIDFromBytes(uuidSeed.getBytes(StandardCharsets.UTF_8)).toString();
+ }
+
+ private static int getComponentCount(final VersionedProcessGroup group, final ComponentType componentType) {
+ final Collection<?> components = switch (componentType) {
+ case PROCESSOR -> group.getProcessors();
+ case INPUT_PORT -> group.getInputPorts();
+ case OUTPUT_PORT -> group.getOutputPorts();
+ case CONNECTION -> group.getConnections();
+ case FUNNEL -> group.getFunnels();
+ case LABEL -> group.getLabels();
+ case PROCESS_GROUP -> group.getProcessGroups();
+ case CONTROLLER_SERVICE -> group.getControllerServices();
+ default -> List.of();
+ };
+
+ return components == null ? 0 : components.size();
+ }
+
+ public static VersionedProcessor addProcessor(final VersionedProcessGroup group, final String processorType, final Bundle bundle, final String name, final Position position) {
+ final VersionedProcessor processor = new VersionedProcessor();
+
+ // Generate deterministic UUID based on group and component type
+ processor.setIdentifier(generateDeterministicUuid(group, ComponentType.PROCESSOR));
+
+ processor.setName(name);
+ processor.setType(processorType);
+ processor.setPosition(position);
+ processor.setBundle(bundle);
+
+ // Set default processor configuration
+ processor.setProperties(new HashMap<>());
+ processor.setPropertyDescriptors(new HashMap<>());
+ processor.setStyle(new HashMap<>());
+ processor.setSchedulingPeriod("0 sec");
+ processor.setSchedulingStrategy("TIMER_DRIVEN");
+ processor.setExecutionNode("ALL");
+ processor.setPenaltyDuration("30 sec");
+ processor.setYieldDuration("1 sec");
+ processor.setBulletinLevel("WARN");
+ processor.setRunDurationMillis(25L);
+ processor.setConcurrentlySchedulableTaskCount(1);
+ processor.setAutoTerminatedRelationships(new HashSet<>());
+ processor.setScheduledState(ScheduledState.ENABLED);
+ processor.setRetryCount(10);
+ processor.setRetriedRelationships(new HashSet<>());
+ processor.setBackoffMechanism("PENALIZE_FLOWFILE");
+ processor.setMaxBackoffPeriod("10 mins");
+ processor.setComponentType(ComponentType.PROCESSOR);
+ processor.setGroupIdentifier(group.getIdentifier());
+
+ group.getProcessors().add(processor);
+ return processor;
+ }
+
+ public static VersionedControllerService addControllerService(final VersionedProcessGroup group, final String serviceType, final Bundle bundle, final String name) {
+ final VersionedControllerService controllerService = new VersionedControllerService();
+
+ // Generate deterministic UUID based on group and component type
+ controllerService.setIdentifier(generateDeterministicUuid(group, ComponentType.CONTROLLER_SERVICE));
+
+ controllerService.setName(name);
+ controllerService.setType(serviceType);
+ controllerService.setBundle(bundle);
+ controllerService.setComponentType(ComponentType.CONTROLLER_SERVICE);
+
+ // Set default controller service configuration
+ controllerService.setProperties(new HashMap<>());
+ controllerService.setPropertyDescriptors(new HashMap<>());
+ controllerService.setControllerServiceApis(new ArrayList<>());
+ controllerService.setAnnotationData(null);
+ controllerService.setScheduledState(ScheduledState.DISABLED);
+ controllerService.setBulletinLevel("WARN");
+ controllerService.setComments(null);
+ controllerService.setGroupIdentifier(group.getIdentifier());
+
+ // Initialize controller services collection if it doesn't exist
+ Set<VersionedControllerService> controllerServices = group.getControllerServices();
+ if (controllerServices == null) {
+ controllerServices = new HashSet<>();
+ group.setControllerServices(controllerServices);
+ }
+ controllerServices.add(controllerService);
+
+ return controllerService;
+ }
+
+ public static Set<VersionedControllerService> getReferencedControllerServices(final VersionedProcessGroup group) {
+ final Set<VersionedControllerService> referencedServices = new HashSet<>();
+ collectReferencedControllerServices(group, referencedServices);
+ return referencedServices;
+ }
+
+ private static void collectReferencedControllerServices(final VersionedProcessGroup group, final Set<VersionedControllerService> referencedServices) {
+ final Map<String, VersionedControllerService> serviceMap = new HashMap<>();
+ for (final VersionedControllerService service : group.getControllerServices()) {
+ serviceMap.put(service.getIdentifier(), service);
+ }
+
+ for (final VersionedProcessor processor : group.getProcessors()) {
+ for (final String propertyValue : processor.getProperties().values()) {
+ final VersionedControllerService referencedService = serviceMap.get(propertyValue);
+ if (referencedService != null) {
+ referencedServices.add(referencedService);
+ }
+ }
+ }
+
+ while (true) {
+ final Set<VersionedControllerService> newlyAddedServices = new HashSet<>();
+
+ for (final VersionedControllerService service : referencedServices) {
+ for (final String propertyValue : service.getProperties().values()) {
+ final VersionedControllerService referencedService = serviceMap.get(propertyValue);
+ if (referencedService != null && !referencedServices.contains(referencedService)) {
+ newlyAddedServices.add(referencedService);
+ }
+ }
+ }
+
+ referencedServices.addAll(newlyAddedServices);
+ if (newlyAddedServices.isEmpty()) {
+ break;
+ }
+ }
+
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ collectReferencedControllerServices(childGroup, referencedServices);
+ }
+ }
+
+ public static void removeControllerServiceReferences(final VersionedProcessGroup processGroup, final String serviceIdentifier) {
+ for (final VersionedProcessor processor : processGroup.getProcessors()) {
+ removeValuesFromMap(processor.getProperties(), serviceIdentifier);
+ }
+
+ for (final VersionedControllerService service : processGroup.getControllerServices()) {
+ removeValuesFromMap(service.getProperties(), serviceIdentifier);
+ }
+
+ for (final VersionedProcessGroup childGroup : processGroup.getProcessGroups()) {
+ removeControllerServiceReferences(childGroup, serviceIdentifier);
+ }
+ }
+
+ private static void removeValuesFromMap(final Map<String, String> properties, final String valueToRemove) {
+ final List<String> keysToRemove = new ArrayList<>();
+ for (final Map.Entry<String, String> entry : properties.entrySet()) {
+ if (Objects.equals(entry.getValue(), valueToRemove)) {
+ keysToRemove.add(entry.getKey());
+ }
+ }
+
+ keysToRemove.forEach(properties::remove);
+ }
+
+ public static void setParameterValue(final VersionedExternalFlow externalFlow, final String parameterName, final String parameterValue) {
+ for (final VersionedParameterContext context : externalFlow.getParameterContexts().values()) {
+ setParameterValue(context, parameterName, parameterValue);
+ }
+ }
+
+ public static void setParameterValue(final VersionedParameterContext parameterContext, final String parameterName, final String parameterValue) {
+ final Set<VersionedParameter> parameters = parameterContext.getParameters();
+ for (final VersionedParameter parameter : parameters) {
+ if (parameter.getName().equals(parameterName)) {
+ parameter.setValue(parameterValue);
+ }
+ }
+ }
+
+ public static void removeUnreferencedControllerServices(final VersionedProcessGroup processGroup) {
+ final Set<VersionedControllerService> referencedServices = getReferencedControllerServices(processGroup);
+ final Set<String> referencedServiceIds = new HashSet<>();
+ for (final VersionedControllerService service : referencedServices) {
+ referencedServiceIds.add(service.getIdentifier());
+ }
+
+ removeUnreferencedControllerServices(processGroup, referencedServiceIds);
+ }
+
+ private static void removeUnreferencedControllerServices(final VersionedProcessGroup processGroup, final Set<String> referencedServiceIds) {
+ processGroup.getControllerServices().removeIf(service -> !referencedServiceIds.contains(service.getIdentifier()));
+
+ for (final VersionedProcessGroup childGroup : processGroup.getProcessGroups()) {
+ removeUnreferencedControllerServices(childGroup, referencedServiceIds);
+ }
+ }
+}
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index b9456a98a0..21e18ce91e 100644
--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -136,6 +136,9 @@ public class NiFiProperties extends ApplicationProperties {
public static final String STATUS_REPOSITORY_QUESTDB_PERSIST_BATCH_SIZE = "nifi.status.repository.questdb.persist.batchsize";
public static final String STATUS_REPOSITORY_QUESTDB_PERSIST_FREQUENCY = "nifi.status.repository.questdb.persist.frequency";
+ // Connector Repository properties
+ public static final String CONNECTOR_REPOSITORY_IMPLEMENTATION = "nifi.components.connectors.repository.implementation";
+
// security properties
public static final String SECURITY_KEYSTORE = "nifi.security.keystore";
public static final String SECURITY_KEYSTORE_TYPE = "nifi.security.keystoreType";
@@ -1811,8 +1814,28 @@ public class NiFiProperties extends ApplicationProperties {
// The Properties(Properties) constructor does NOT inherit the provided values, just uses them as default values
if (additionalProperties != null) {
- additionalProperties.forEach(properties::put);
+ properties.putAll(additionalProperties);
+ }
+
+ return createNiFiProperties(properties);
+ }
+
+ public static NiFiProperties createBasicNiFiProperties(final InputStream inputStream) {
+ return createBasicNiFiProperties(inputStream, null);
+ }
+
+ public static NiFiProperties createBasicNiFiProperties(final InputStream inputStream, final Properties additionalProperties) {
+ final Properties properties = new Properties();
+ readFromInputStream(inputStream, properties);
+
+ if (additionalProperties != null) {
+ properties.putAll(additionalProperties);
}
+
+ return createNiFiProperties(properties);
+ }
+
+ private static NiFiProperties createNiFiProperties(final Properties properties) {
return new NiFiProperties() {
@Override
public String getProperty(String key) {
@@ -1832,9 +1855,7 @@ public class NiFiProperties extends ApplicationProperties {
}
private static void readFromPropertiesFile(String propertiesFilePath, Properties properties) {
- final String nfPropertiesFilePath = (propertiesFilePath == null)
- ? System.getProperty(PROPERTIES_FILE_PATH)
- : propertiesFilePath;
+ final String nfPropertiesFilePath = (propertiesFilePath == null) ? System.getProperty(PROPERTIES_FILE_PATH) : propertiesFilePath;
if (nfPropertiesFilePath != null) {
final File propertiesFile = new File(nfPropertiesFilePath.trim());
if (!propertiesFile.exists()) {
@@ -1845,24 +1866,25 @@ public class NiFiProperties extends ApplicationProperties {
throw new RuntimeException("Properties file exists but cannot be read '"
+ propertiesFile.getAbsolutePath() + "'");
}
- InputStream inStream = null;
- try {
- inStream = new BufferedInputStream(new FileInputStream(propertiesFile));
+
+ try (final InputStream fileIn = new FileInputStream(propertiesFile);
+ final InputStream inStream = new BufferedInputStream(fileIn)) {
+
properties.load(inStream);
} catch (final Exception ex) {
- throw new RuntimeException("Cannot load properties file due to "
- + ex.getLocalizedMessage(), ex);
- } finally {
- if (null != inStream) {
- try {
- inStream.close();
- } catch (final Exception ignored) {
- }
- }
+ throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
}
}
}
+ private static void readFromInputStream(final InputStream inputStream, final Properties properties) {
+ try {
+ properties.load(inputStream);
+ } catch (final Exception ex) {
+ throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
+ }
+ }
+
/**
* This method is used to validate the NiFi properties when the file is loaded
* for the first time. The objective is to stop NiFi startup in case a property
diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml
index f6b2bd57ba..862a72346a 100644
--- a/nifi-commons/pom.xml
+++ b/nifi-commons/pom.xml
@@ -68,6 +68,7 @@
<module>nifi-write-ahead-log</module>
<module>nifi-xml-processing</module>
<module>nifi-calcite-utils</module>
+ <module>nifi-connector-utils</module>
</modules>
<properties>
<guava.version>33.5.0-jre</guava.version>
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-api/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-api/pom.xml
new file mode 100644
index 0000000000..2d581f17da
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-server-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorConfigVerificationResult.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorConfigVerificationResult.java
index 96806fca1d..9ec3ac2878 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorConfigVerificationResult.java
@@ -14,23 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.components.ConfigVerificationResult;
-public interface ValidationContextFactory {
+import java.util.List;
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+public interface ConnectorConfigVerificationResult {
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ List<ConfigVerificationResult> getAllResults();
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ List<ConfigVerificationResult> getFailedResults();
+
+ void assertNoFailures();
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorMockServer.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorMockServer.java
index 96806fca1d..7b24c973c9 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorMockServer.java
@@ -14,23 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.NiFiServer;
+import org.apache.nifi.processor.Processor;
-public interface ValidationContextFactory {
+import java.io.File;
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+public interface ConnectorMockServer extends NiFiServer, ConnectorTestRunner {
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ void instantiateConnector(String connectorClassName);
+
+ void registerMockBundle(ClassLoader classLoader, File workingDirectory);
+
+ void mockProcessor(String processorType, Class<? extends Processor> mockProcessorClass);
- Optional<FlowAnalyzer> getFlowAnalyzer();
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java
new file mode 100644
index 0000000000..798f6d4676
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.List;
+
+public interface ConnectorTestRunner extends Closeable {
+
+ void applyUpdate() throws FlowUpdateException;
+
+ default void configure(String stepName, PropertyGroupConfiguration groupConfiguration) throws FlowUpdateException {
+ configure(stepName, List.of(groupConfiguration));
+ }
+
+ void configure(String stepName, List<PropertyGroupConfiguration> groupConfigurations) throws FlowUpdateException;
+
+ ConnectorConfigVerificationResult verifyConfiguration(String stepName, List<PropertyGroupConfiguration> groupConfigurations);
+
+ void startConnector();
+
+ void stopConnector();
+
+ void waitForDataIngested(Duration maxWaitTime);
+
+ void waitForIdle(Duration maxWaitTime);
+
+ void waitForIdle(Duration minimumIdleTime, Duration maxWaitTime);
+
+ List<ValidationResult> validate();
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml
new file mode 100644
index 0000000000..eb4b1e8f72
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-server-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-server</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <repositories>
+ <!-- Shibboleth Repository required for OpenSAML -->
+ <!-- This is required because nifi-framework-nar is the parent/NAR Dependency, andi t requires shibboleth. -->
+ <repository>
+ <id>shibboleth</id>
+ <url>https://build.shibboleth.net/nexus/content/repositories/releases/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+</project>
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-server/pom.xml
new file mode 100644
index 0000000000..040596e620
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-server</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-core-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-core</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-encryptor</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-server-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java
new file mode 100644
index 0000000000..3aa7bbc543
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.service.AuditService;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.HistoryQuery;
+import org.apache.nifi.history.PreviousValue;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class MockAuditService implements AuditService {
+ @Override
+ public void addActions(final Collection<Action> actions) {
+ }
+
+ @Override
+ public Map<String, List<PreviousValue>> getPreviousValues(final String componentId) {
+ return Map.of();
+ }
+
+ @Override
+ public void deletePreviousValues(final String propertyName, final String componentId) {
+ }
+
+ @Override
+ public History getActions(final HistoryQuery actionQuery) {
+ return null;
+ }
+
+ @Override
+ public History getActions(final int firstActionId, final int maxActions) {
+ return null;
+ }
+
+ @Override
+ public Action getAction(final Integer actionId) {
+ return null;
+ }
+
+ @Override
+ public void purgeActions(final Date end, final Action purgeAction) {
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorInitializationContext.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorInitializationContext.java
new file mode 100644
index 0000000000..38dcec3474
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorInitializationContext.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.asset.AssetManager;
+import org.apache.nifi.components.connector.FrameworkConnectorInitializationContextBuilder;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.FrameworkConnectorInitializationContext;
+import org.apache.nifi.components.connector.FrameworkFlowContext;
+import org.apache.nifi.components.connector.SecretsManager;
+import org.apache.nifi.components.connector.components.FlowContext;
+import org.apache.nifi.flow.VersionedExternalFlow;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.flow.VersionedProcessor;
+import org.apache.nifi.logging.ComponentLog;
+
+public class MockConnectorInitializationContext implements FrameworkConnectorInitializationContext {
+
+ private final String identifier;
+ private final String name;
+ private final ComponentLog componentLog;
+ private final SecretsManager secretsManager;
+ private final AssetManager assetManager;
+
+ private final MockExtensionMapper mockExtensionMapper;
+
+ private MockConnectorInitializationContext(final Builder builder) {
+ this.identifier = builder.identifier;
+ this.name = builder.name;
+ this.componentLog = builder.componentLog;
+ this.secretsManager = builder.secretsManager;
+ this.assetManager = builder.assetManager;
+ this.mockExtensionMapper = builder.mockExtensionMapper;
+ }
+
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public ComponentLog getLogger() {
+ return componentLog;
+ }
+
+ @Override
+ public SecretsManager getSecretsManager() {
+ return secretsManager;
+ }
+
+ @Override
+ public AssetManager getAssetManager() {
+ return assetManager;
+ }
+
+ @Override
+ public void updateFlow(final FlowContext flowContext, final VersionedExternalFlow versionedExternalFlow) throws FlowUpdateException {
+ if (!(flowContext instanceof final FrameworkFlowContext frameworkFlowContext)) {
+ throw new IllegalArgumentException("FlowContext is not an instance provided by the framework");
+ }
+
+ replaceMocks(versionedExternalFlow.getFlowContents());
+
+ // TODO: Probably should eliminate this method and instead move AssetManager to the FlowContext and add a method there
+ // to update the flow.
+ frameworkFlowContext.updateFlow(versionedExternalFlow, assetManager);
+ }
+
+ private void replaceMocks(final VersionedProcessGroup group) {
+ if (group.getProcessors() != null) {
+ for (final VersionedProcessor processor : group.getProcessors()) {
+ mockExtensionMapper.mapProcessor(processor);
+ }
+ }
+
+ if (group.getProcessGroups() != null) {
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ replaceMocks(childGroup);
+ }
+ }
+ }
+
+ private void updateParameterContext(final VersionedProcessGroup group, final String parameterContextName) {
+ group.setParameterContextName(parameterContextName);
+ if (group.getProcessGroups() != null) {
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ updateParameterContext(childGroup, parameterContextName);
+ }
+ }
+ }
+
+
+ public static class Builder implements FrameworkConnectorInitializationContextBuilder {
+ private final MockExtensionMapper mockExtensionMapper;
+ private String identifier;
+ private String name;
+ private ComponentLog componentLog;
+ private SecretsManager secretsManager;
+ private AssetManager assetManager;
+
+ public Builder(final MockExtensionMapper mockExtensionMapper) {
+ this.mockExtensionMapper = mockExtensionMapper;
+ }
+
+ public Builder identifier(final String identifier) {
+ this.identifier = identifier;
+ return this;
+ }
+
+ public Builder name(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder componentLog(final ComponentLog componentLog) {
+ this.componentLog = componentLog;
+ return this;
+ }
+
+ public Builder secretsManager(final SecretsManager secretsManager) {
+ this.secretsManager = secretsManager;
+ return this;
+ }
+
+ public Builder assetManager(final AssetManager assetManager) {
+ this.assetManager = assetManager;
+ return this;
+ }
+
+ public MockConnectorInitializationContext build() {
+ return new MockConnectorInitializationContext(this);
+ }
+
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorRepository.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorRepository.java
index 96806fca1d..9f95111674 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorRepository.java
@@ -14,23 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.components.connector.FrameworkConnectorInitializationContextBuilder;
+import org.apache.nifi.components.connector.StandardConnectorRepository;
-public interface ValidationContextFactory {
+public class MockConnectorRepository extends StandardConnectorRepository {
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+ private volatile MockExtensionMapper mockExtensionMapper;
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ public void setMockExtensionMapper(final MockExtensionMapper mapper) {
+ mockExtensionMapper = mapper;
+ }
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ @Override
+ public FrameworkConnectorInitializationContextBuilder createInitializationContextBuilder() {
+ return new MockConnectorInitializationContext.Builder(mockExtensionMapper);
+ }
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java
new file mode 100644
index 0000000000..7af2240d2d
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.bundle.BundleCoordinate;
+import org.apache.nifi.bundle.BundleDetails;
+import org.apache.nifi.nar.InstanceClassLoader;
+import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
+import org.apache.nifi.processor.Processor;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class MockExtensionDiscoveringManager extends StandardExtensionDiscoveringManager {
+ private final ConcurrentMap<String, ClassLoader> mockComponentClassLoaders = new ConcurrentHashMap<>();
+
+ public synchronized void addProcessor(final Class<? extends Processor> mockProcessorClass) {
+ final BundleDetails bundleDetails = new BundleDetails.Builder()
+ .workingDir(new File("target/work/extensions/mock-bundle"))
+ .coordinate(new BundleCoordinate("org.apache.nifi.mock", mockProcessorClass.getName(), "1.0.0"))
+ .build();
+
+ final Bundle mockBundle = new Bundle(bundleDetails, mockProcessorClass.getClassLoader());
+ discoverExtensions(Set.of(mockBundle));
+
+ mockComponentClassLoaders.put(mockProcessorClass.getName(), mockProcessorClass.getClassLoader());
+ registerExtensionClass(Processor.class, mockProcessorClass.getName(), mockBundle);
+ }
+
+ @Override
+ public InstanceClassLoader createInstanceClassLoader(final String classType, final String instanceIdentifier, final Bundle bundle, final Set<URL> additionalUrls,
+ final boolean register, final String classloaderIsolationKey) {
+
+ final ClassLoader classLoader = mockComponentClassLoaders.get(classType);
+ if (classLoader != null) {
+ return new InstanceClassLoader(instanceIdentifier, classType, additionalUrls, Set.of(), classLoader);
+ }
+
+ return super.createInstanceClassLoader(classType, instanceIdentifier, bundle, additionalUrls, register, classloaderIsolationKey);
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java
new file mode 100644
index 0000000000..ed532cc253
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.flow.Bundle;
+import org.apache.nifi.flow.VersionedProcessor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MockExtensionMapper {
+
+ private final Map<String, String> processorMocks = new HashMap<>();
+
+ public void mockProcessor(final String processorType, final String mockProcessorClassName) {
+ processorMocks.put(processorType, mockProcessorClassName);
+ }
+
+ public void mapProcessor(final VersionedProcessor processor) {
+ final String type = processor.getType();
+ final String implementationClassName = processorMocks.get(type);
+ if (implementationClassName == null) {
+ return;
+ }
+
+ processor.setType(implementationClassName);
+ processor.setBundle(new Bundle("org.apache.nifi.mock", implementationClassName, "1.0.0"));
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockRuleViolationsManager.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockRuleViolationsManager.java
new file mode 100644
index 0000000000..d7f590bb9f
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockRuleViolationsManager.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.flow.VersionedComponent;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.validation.RuleViolation;
+import org.apache.nifi.validation.RuleViolationsManager;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class MockRuleViolationsManager implements RuleViolationsManager {
+ @Override
+ public void upsertComponentViolations(final String subjectId, final Collection<RuleViolation> violations) {
+
+ }
+
+ @Override
+ public void upsertGroupViolations(final VersionedProcessGroup processGroup, final Collection<RuleViolation> violations,
+ final Map<VersionedComponent, Collection<RuleViolation>> componentToRuleViolations) {
+
+ }
+
+ @Override
+ public Collection<RuleViolation> getRuleViolationsForSubject(final String subjectId) {
+ return List.of();
+ }
+
+ @Override
+ public Collection<RuleViolation> getRuleViolationsForGroup(final String groupId) {
+ return List.of();
+ }
+
+ @Override
+ public Collection<RuleViolation> getRuleViolationsForGroups(final Collection<String> groupIds) {
+ return List.of();
+ }
+
+ @Override
+ public Collection<RuleViolation> getAllRuleViolations() {
+ return List.of();
+ }
+
+ @Override
+ public void removeRuleViolationsForSubject(final String subjectId) {
+
+ }
+
+ @Override
+ public void removeRuleViolationsForRule(final String ruleId) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockServerConfigVerificationResult.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockServerConfigVerificationResult.java
new file mode 100644
index 0000000000..fed82f9290
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockServerConfigVerificationResult.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
+
+import java.util.Collections;
+import java.util.List;
+
+public class MockServerConfigVerificationResult implements ConnectorConfigVerificationResult {
+ private final List<ConfigVerificationResult> results;
+
+ public MockServerConfigVerificationResult(final List<ConfigVerificationResult> results) {
+ this.results = results;
+ }
+
+ @Override
+ public List<ConfigVerificationResult> getAllResults() {
+ return Collections.unmodifiableList(results);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> getFailedResults() {
+ return results.stream()
+ .filter(result -> result.getOutcome() == Outcome.FAILED)
+ .toList();
+ }
+
+ @Override
+ public void assertNoFailures() {
+ final List<ConfigVerificationResult> failedResults = getFailedResults();
+ if (!failedResults.isEmpty()) {
+ throw new AssertionError("Configuration verification failed: " + failedResults);
+ }
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManager.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManager.java
new file mode 100644
index 0000000000..f22269ceab
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManager.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.Scope;
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.components.state.StateMap;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MockStateManager implements StateManager {
+ private final AtomicLong versionCounter = new AtomicLong(0L);
+ private final boolean dropStateKeySupported;
+
+ private volatile StateMap localStateMap = new MockStateMap(null, -1L);
+ private volatile StateMap clusterStateMap = new MockStateMap(null, -1L);
+
+ public MockStateManager(final boolean dropStateKeySupported) {
+ this.dropStateKeySupported = dropStateKeySupported;
+ }
+
+ @Override
+ public synchronized void setState(final Map<String, String> state, final Scope scope) throws IOException {
+ final StateMap newStateMap = new MockStateMap(state, versionCounter.incrementAndGet());
+
+ if (scope == Scope.CLUSTER) {
+ clusterStateMap = newStateMap;
+ } else {
+ localStateMap = newStateMap;
+ }
+ }
+
+ @Override
+ public synchronized StateMap getState(final Scope scope) throws IOException {
+ if (scope == Scope.CLUSTER) {
+ return clusterStateMap;
+ } else {
+ return localStateMap;
+ }
+ }
+
+ @Override
+ public synchronized boolean replace(final StateMap oldValue, final Map<String, String> newValue, final Scope scope) throws IOException {
+ final StateMap currentStateMap = scope == Scope.CLUSTER ? clusterStateMap : localStateMap;
+
+ if (currentStateMap == oldValue) {
+ final StateMap newStateMap = new MockStateMap(newValue, versionCounter.incrementAndGet());
+
+ if (scope == Scope.CLUSTER) {
+ clusterStateMap = newStateMap;
+ } else {
+ localStateMap = newStateMap;
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public synchronized void clear(final Scope scope) throws IOException {
+ setState(Collections.emptyMap(), scope);
+ }
+
+ @Override
+ public boolean isStateKeyDropSupported() {
+ return dropStateKeySupported;
+ }
+}
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManagerProvider.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManagerProvider.java
new file mode 100644
index 0000000000..77484904f2
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManagerProvider.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.components.state.StateManagerProvider;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class MockStateManagerProvider implements StateManagerProvider {
+ private final ConcurrentMap<String, StateManager> stateManagers = new ConcurrentHashMap<>();
+ private volatile boolean clusterProviderEnabled = false;
+
+ @Override
+ public StateManager getStateManager(final String componentId, final boolean dropStateKeySupported) {
+ return stateManagers.computeIfAbsent(componentId, id -> new MockStateManager(dropStateKeySupported));
+ }
+
+ @Override
+ public void onComponentRemoved(final String componentId) {
+ stateManagers.remove(componentId);
+ }
+
+ @Override
+ public void shutdown() {
+ stateManagers.clear();
+ }
+
+ @Override
+ public void enableClusterProvider() {
+ clusterProviderEnabled = true;
+ }
+
+ @Override
+ public void disableClusterProvider() {
+ clusterProviderEnabled = false;
+ }
+
+ @Override
+ public boolean isClusterProviderEnabled() {
+ return clusterProviderEnabled;
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateMap.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateMap.java
new file mode 100644
index 0000000000..54b0980d61
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateMap.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.StateMap;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class MockStateMap implements StateMap {
+ private final Map<String, String> stateValues;
+ private final long version;
+
+ public MockStateMap(final Map<String, String> stateValues, final long version) {
+ this.stateValues = stateValues == null ? Collections.emptyMap() : new HashMap<>(stateValues);
+ this.version = version;
+ }
+
+ @Override
+ public Optional<String> getStateVersion() {
+ return version == -1L ? Optional.empty() : Optional.of(Long.toString(version));
+ }
+
+ @Override
+ public String get(final String key) {
+ return stateValues.get(key);
+ }
+
+ @Override
+ public Map<String, String> toMap() {
+ return Collections.unmodifiableMap(stateValues);
+ }
+
+ @Override
+ public String toString() {
+ return "MockStateMap[version=" + version + ", values=" + stateValues + "]";
+ }
+}
+
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/NopPropertyEncryptor.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/NopPropertyEncryptor.java
index 96806fca1d..56fbe816b7 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/NopPropertyEncryptor.java
@@ -14,23 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.encrypt.PropertyEncryptor;
-public interface ValidationContextFactory {
+public class NopPropertyEncryptor implements PropertyEncryptor {
+ @Override
+ public String encrypt(final String property) {
+ return property;
+ }
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
-
- Optional<RuleViolationsManager> getRuleViolationsManager();
-
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ @Override
+ public String decrypt(final String encryptedProperty) {
+ return encryptedProperty;
+ }
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/PermitAllAuthorizer.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/PermitAllAuthorizer.java
new file mode 100644
index 0000000000..75b2577f25
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/PermitAllAuthorizer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.authorization.AuthorizationRequest;
+import org.apache.nifi.authorization.AuthorizationResult;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.authorization.AuthorizerConfigurationContext;
+import org.apache.nifi.authorization.AuthorizerInitializationContext;
+import org.apache.nifi.authorization.exception.AuthorizationAccessException;
+import org.apache.nifi.authorization.exception.AuthorizerCreationException;
+import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
+
+public class PermitAllAuthorizer implements Authorizer {
+ @Override
+ public AuthorizationResult authorize(final AuthorizationRequest request) throws AuthorizationAccessException {
+ return AuthorizationResult.approved();
+ }
+
+ @Override
+ public void initialize(final AuthorizerInitializationContext initializationContext) throws AuthorizerCreationException {
+ }
+
+ @Override
+ public void onConfigured(final AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException {
+ }
+
+ @Override
+ public void preDestruction() throws AuthorizerDestructionException {
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/StandardConnectorMockServer.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/StandardConnectorMockServer.java
new file mode 100644
index 0000000000..1b8034c9ff
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/StandardConnectorMockServer.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.admin.service.AuditService;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.bundle.BundleCoordinate;
+import org.apache.nifi.bundle.BundleDetails;
+import org.apache.nifi.cluster.ClusterDetailsFactory;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.ConnectorNode;
+import org.apache.nifi.components.connector.ConnectorRepository;
+import org.apache.nifi.components.connector.ConnectorState;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.components.connector.StandaloneConnectorRequestReplicator;
+import org.apache.nifi.components.state.StateManagerProvider;
+import org.apache.nifi.components.validation.DisabledServiceValidationResult;
+import org.apache.nifi.components.validation.ValidationState;
+import org.apache.nifi.connectable.FlowFileTransferCounts;
+import org.apache.nifi.controller.DecommissionTask;
+import org.apache.nifi.controller.FlowController;
+import org.apache.nifi.controller.repository.FlowFileEventRepository;
+import org.apache.nifi.controller.repository.metrics.RingBufferEventRepository;
+import org.apache.nifi.controller.status.history.StatusHistoryDumpFactory;
+import org.apache.nifi.controller.status.history.StatusHistoryRepository;
+import org.apache.nifi.controller.status.history.VolatileComponentStatusRepository;
+import org.apache.nifi.diagnostics.DiagnosticsFactory;
+import org.apache.nifi.encrypt.PropertyEncryptor;
+import org.apache.nifi.engine.FlowEngine;
+import org.apache.nifi.events.VolatileBulletinRepository;
+import org.apache.nifi.nar.ExtensionMapping;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.reporting.BulletinRepository;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.validation.RuleViolationsManager;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public class StandardConnectorMockServer implements ConnectorMockServer {
+ private static final String CONNECTOR_ID = "test-connector";
+
+ private Bundle systemBundle;
+ private Set<Bundle> bundles;
+ private NiFiProperties nifiProperties;
+ private FlowController flowController;
+ private MockExtensionDiscoveringManager extensionManager;
+ private ConnectorNode connectorNode;
+ private FlowEngine flowEngine;
+ private MockExtensionMapper mockExtensionMapper;
+ private FlowFileTransferCounts initialFlowFileTransferCounts = new FlowFileTransferCounts(0L, 0L, 0L, 0L);
+
+ @Override
+ public void start() {
+ extensionManager = new MockExtensionDiscoveringManager();
+ extensionManager.discoverExtensions(systemBundle, bundles);
+ extensionManager.logClassLoaderMapping();
+
+ final FlowFileEventRepository flowFileEventRepository = new RingBufferEventRepository(5);
+ final Authorizer authorizer = new PermitAllAuthorizer();
+ final AuditService auditService = new MockAuditService();
+ final PropertyEncryptor propertyEncryptor = new NopPropertyEncryptor();
+ final BulletinRepository bulletinRepository = new VolatileBulletinRepository();
+ final StatusHistoryRepository statusHistoryRepository = new VolatileComponentStatusRepository(nifiProperties);
+ final RuleViolationsManager ruleViolationManager = new MockRuleViolationsManager();
+ final StateManagerProvider stateManagerProvider = new MockStateManagerProvider();
+
+ flowController = FlowController.createStandaloneInstance(
+ flowFileEventRepository,
+ null,
+ nifiProperties,
+ authorizer,
+ auditService,
+ propertyEncryptor,
+ bulletinRepository,
+ extensionManager,
+ statusHistoryRepository,
+ ruleViolationManager,
+ stateManagerProvider,
+ new StandaloneConnectorRequestReplicator());
+
+ try {
+ flowController.getRepositoryContextFactory().getFlowFileRepository().loadFlowFiles(Collections::emptyList);
+ } catch (final IOException e) {
+ throw new RuntimeException("Failed to initialize FlowFile Repository", e);
+ }
+
+ final ConnectorRepository connectorRepository = flowController.getConnectorRepository();
+ if (!(connectorRepository instanceof MockConnectorRepository)) {
+ throw new IllegalStateException("Connector Repository is not an instance of MockConnectorRepository");
+ }
+
+ mockExtensionMapper = new MockExtensionMapper();
+ ((MockConnectorRepository) connectorRepository).setMockExtensionMapper(mockExtensionMapper);
+
+ flowEngine = new FlowEngine(4, "Connector Threads");
+ }
+
+ @Override
+ public void initialize(final NiFiProperties properties, final Bundle systemBundle, final Set<Bundle> bundles, final ExtensionMapping extensionMapping) {
+ this.systemBundle = systemBundle;
+ this.bundles = bundles;
+ this.nifiProperties = properties;
+ }
+
+ @Override
+ public void stop() {
+ if (flowEngine != null) {
+ flowEngine.shutdown();
+ }
+ if (flowController != null) {
+ flowController.shutdown(false);
+ }
+ }
+
+ @Override
+ public DiagnosticsFactory getDiagnosticsFactory() {
+ return null;
+ }
+
+ @Override
+ public DiagnosticsFactory getThreadDumpFactory() {
+ return null;
+ }
+
+ @Override
+ public DecommissionTask getDecommissionTask() {
+ return null;
+ }
+
+ @Override
+ public ClusterDetailsFactory getClusterDetailsFactory() {
+ return null;
+ }
+
+ @Override
+ public StatusHistoryDumpFactory getStatusHistoryDumpFactory() {
+ return null;
+ }
+
+ @Override
+ public void instantiateConnector(final String connectorClassName) {
+ final List<Bundle> bundles = extensionManager.getBundles(connectorClassName);
+ if (bundles.isEmpty()) {
+ throw new IllegalStateException("No bundles found for connector class: " + connectorClassName + " - ensure that you have included all relevant NARs in the configured lib directory");
+ }
+ if (bundles.size() > 1) {
+ throw new IllegalStateException("Multiple bundles found for connector class: " + connectorClassName + " - unable to determine which bundle to use. Ensure that only a single version of " +
+ "the Connector is included in the configured lib directory. Available bundles: " + bundles);
+ }
+
+ final BundleCoordinate bundleCoordinate = bundles.getFirst().getBundleDetails().getCoordinate();
+ connectorNode = flowController.getFlowManager().createConnector(connectorClassName, CONNECTOR_ID, bundleCoordinate, true, true);
+ }
+
+ @Override
+ public void registerMockBundle(final ClassLoader classLoader, final File workingDirectory) {
+ final BundleDetails mockBundleDetails = new BundleDetails.Builder()
+ .workingDir(workingDirectory)
+ .coordinate(new BundleCoordinate("org.apache.nifi.mock", "nifi-connector-mock-bundle", "1.0.0"))
+ .build();
+ }
+
+ @Override
+ public void applyUpdate() throws FlowUpdateException {
+ final ConnectorState initialDesiredState = connectorNode.getDesiredState();
+
+ connectorNode.prepareForUpdate();
+ connectorNode.applyUpdate();
+
+ if (initialDesiredState == ConnectorState.RUNNING) {
+ connectorNode.start(flowEngine);
+ } else {
+ connectorNode.stop(flowEngine);
+ }
+ }
+
+ @Override
+ public void configure(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) throws FlowUpdateException {
+ connectorNode.setConfiguration(stepName, groupConfigurations);
+ }
+
+ @Override
+ public ConnectorConfigVerificationResult verifyConfiguration(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) {
+ final List<ConfigVerificationResult> results = connectorNode.verifyConfigurationStep(stepName, groupConfigurations);
+ return new MockServerConfigVerificationResult(results);
+ }
+
+ @Override
+ public void startConnector() {
+ initialFlowFileTransferCounts = connectorNode.getFlowFileTransferCounts();
+
+ connectorNode.start(flowEngine);
+ }
+
+ @Override
+ public void stopConnector() {
+ try {
+ connectorNode.stop(flowEngine).get(10, TimeUnit.SECONDS);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for connector to stop", e);
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to stop Connector", e);
+ }
+ }
+
+ @Override
+ public void waitForDataIngested(final Duration maxWaitTime) {
+ final long startTime = System.currentTimeMillis();
+ final long expirationTime = startTime + maxWaitTime.toMillis();
+
+ while (getTransferCountsSinceStart().getReceivedCount() == 0L) {
+ if (System.currentTimeMillis() > expirationTime) {
+ throw new RuntimeException("Timed out waiting for data to be ingested by the Connector");
+ }
+
+ try {
+ Thread.sleep(100L);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for data to be ingested by the Connector", e);
+ }
+ }
+ }
+
+ private FlowFileTransferCounts getTransferCountsSinceStart() {
+ final FlowFileTransferCounts currentCounts = connectorNode.getFlowFileTransferCounts();
+ return currentCounts.minus(initialFlowFileTransferCounts);
+ }
+
+ @Override
+ public void waitForIdle(final Duration maxWaitTime) {
+ waitForIdle(Duration.ofMillis(0L), maxWaitTime);
+ }
+
+ @Override
+ public void waitForIdle(final Duration minimumIdleTime, final Duration maxWaitTime) {
+ Optional<Duration> idleTime = connectorNode.getIdleDuration();
+
+ // Wait until idleTime is not empty and is at least equal to minimumIdleTime
+ final long startTime = System.currentTimeMillis();
+ final long expirationTime = startTime + maxWaitTime.toMillis();
+
+ while (idleTime.isEmpty() || idleTime.get().compareTo(minimumIdleTime) <= 0) {
+ if (System.currentTimeMillis() > expirationTime) {
+ throw new RuntimeException("Timed out waiting for Connector to be idle");
+ }
+
+ try {
+ Thread.sleep(100L);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for Connector to be idle", e);
+ }
+
+ idleTime = connectorNode.getIdleDuration();
+ }
+ }
+
+ @Override
+ public List<ValidationResult> validate() {
+ final ValidationState validationState = connectorNode.performValidation();
+ return validationState.getValidationErrors().stream()
+ .filter(result -> !result.isValid())
+ .filter(result -> !DisabledServiceValidationResult.isMatch(result))
+ .toList();
+ }
+
+ @Override
+ public void mockProcessor(final String processorType, final Class<? extends Processor> mockProcessorClass) {
+ mockExtensionMapper.mockProcessor(processorType, mockProcessorClass.getName());
+ extensionManager.addProcessor(mockProcessorClass);
+ }
+
+ @Override
+ public void close() {
+ stop();
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.NiFiServer
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.NiFiServer
index 56740ff7c3..70572a99ea 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.NiFiServer
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.mock.connector.server.StandardConnectorMockServer
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.components.connector.ConnectorRepository
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.components.connector.ConnectorRepository
index 56740ff7c3..d9230afaec 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.components.connector.ConnectorRepository
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.mock.connector.server.MockConnectorRepository
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/test/java/org/apache/nifi/mock/connector/server/MockStateManagerProviderTest.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/test/java/org/apache/nifi/mock/connector/server/MockStateManagerProviderTest.java
new file mode 100644
index 0000000000..d8ecf8e872
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/test/java/org/apache/nifi/mock/connector/server/MockStateManagerProviderTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.Scope;
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.components.state.StateMap;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class MockStateManagerProviderTest {
+
+ @Test
+ void testGetStateManagerReturnsSameInstance() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ final StateManager stateManager2 = provider.getStateManager("component1", false);
+
+ assertNotNull(stateManager1);
+ assertSame(stateManager1, stateManager2);
+ }
+
+ @Test
+ void testGetStateManagerReturnsDifferentInstancesForDifferentComponents() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ final StateManager stateManager2 = provider.getStateManager("component2", false);
+
+ assertNotNull(stateManager1);
+ assertNotNull(stateManager2);
+ assertNotSame(stateManager1, stateManager2);
+ }
+
+ @Test
+ void testOnComponentRemoved() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ assertNotNull(stateManager1);
+
+ provider.onComponentRemoved("component1");
+
+ final StateManager stateManager2 = provider.getStateManager("component1", false);
+ assertNotNull(stateManager2);
+ assertNotSame(stateManager1, stateManager2);
+ }
+
+ @Test
+ void testShutdown() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ provider.getStateManager("component1", false);
+ provider.getStateManager("component2", false);
+
+ provider.shutdown();
+
+ final StateManager stateManager = provider.getStateManager("component1", false);
+ assertNotNull(stateManager);
+ }
+
+ @Test
+ void testClusterProviderEnabled() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ assertFalse(provider.isClusterProviderEnabled());
+
+ provider.enableClusterProvider();
+ assertTrue(provider.isClusterProviderEnabled());
+
+ provider.disableClusterProvider();
+ assertFalse(provider.isClusterProviderEnabled());
+ }
+
+ @Test
+ void testStateManagerSetAndGetLocalState() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state = new HashMap<>();
+ state.put("key1", "value1");
+ state.put("key2", "value2");
+
+ stateManager.setState(state, Scope.LOCAL);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ assertNotNull(stateMap);
+ assertTrue(stateMap.getStateVersion().isPresent());
+ assertEquals("value1", stateMap.get("key1"));
+ assertEquals("value2", stateMap.get("key2"));
+ assertEquals(2, stateMap.toMap().size());
+ }
+
+ @Test
+ void testStateManagerSetAndGetClusterState() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state = new HashMap<>();
+ state.put("key1", "value1");
+ state.put("key2", "value2");
+
+ stateManager.setState(state, Scope.CLUSTER);
+
+ final StateMap stateMap = stateManager.getState(Scope.CLUSTER);
+ assertNotNull(stateMap);
+ assertTrue(stateMap.getStateVersion().isPresent());
+ assertEquals("value1", stateMap.get("key1"));
+ assertEquals("value2", stateMap.get("key2"));
+ }
+
+ @Test
+ void testStateManagerReplace() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> initialState = new HashMap<>();
+ initialState.put("key1", "value1");
+ stateManager.setState(initialState, Scope.LOCAL);
+
+ final StateMap oldStateMap = stateManager.getState(Scope.LOCAL);
+
+ final Map<String, String> newState = new HashMap<>();
+ newState.put("key1", "value2");
+
+ final boolean replaced = stateManager.replace(oldStateMap, newState, Scope.LOCAL);
+ assertTrue(replaced);
+
+ final StateMap updatedStateMap = stateManager.getState(Scope.LOCAL);
+ assertEquals("value2", updatedStateMap.get("key1"));
+
+ final boolean replacedAgain = stateManager.replace(oldStateMap, initialState, Scope.LOCAL);
+ assertFalse(replacedAgain);
+ }
+
+ @Test
+ void testStateManagerClear() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state = new HashMap<>();
+ state.put("key1", "value1");
+ stateManager.setState(state, Scope.LOCAL);
+
+ stateManager.clear(Scope.LOCAL);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ assertNotNull(stateMap);
+ assertTrue(stateMap.toMap().isEmpty());
+ }
+
+ @Test
+ void testStateKeyDropSupported() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ assertFalse(stateManager1.isStateKeyDropSupported());
+
+ final StateManager stateManager2 = provider.getStateManager("component2", true);
+ assertTrue(stateManager2.isStateKeyDropSupported());
+ }
+
+ @Test
+ void testThreadSafety() throws InterruptedException, IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+ final int threadCount = 10;
+ final int operationsPerThread = 100;
+ final ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch completionLatch = new CountDownLatch(threadCount);
+ final AtomicInteger successCount = new AtomicInteger(0);
+
+ for (int i = 0; i < threadCount; i++) {
+ final int threadId = i;
+ executorService.submit(() -> {
+ try {
+ startLatch.await();
+ for (int j = 0; j < operationsPerThread; j++) {
+ final Map<String, String> state = new HashMap<>();
+ state.put("thread", String.valueOf(threadId));
+ state.put("operation", String.valueOf(j));
+ stateManager.setState(state, Scope.LOCAL);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ if (stateMap != null && stateMap.toMap().size() == 2) {
+ successCount.incrementAndGet();
+ }
+ }
+ } catch (final Exception e) {
+ e.printStackTrace();
+ } finally {
+ completionLatch.countDown();
+ }
+ });
+ }
+
+ startLatch.countDown();
+ completionLatch.await(30, TimeUnit.SECONDS);
+ executorService.shutdown();
+
+ assertEquals(threadCount * operationsPerThread, successCount.get());
+ final StateMap finalState = stateManager.getState(Scope.LOCAL);
+ assertNotNull(finalState);
+ assertEquals(2, finalState.toMap().size());
+ }
+
+ @Test
+ void testEmptyStateMap() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ assertNotNull(stateMap);
+ assertFalse(stateMap.getStateVersion().isPresent());
+ assertTrue(stateMap.toMap().isEmpty());
+ }
+
+ @Test
+ void testVersionIncrementing() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state1 = new HashMap<>();
+ state1.put("key", "value1");
+ stateManager.setState(state1, Scope.LOCAL);
+ final StateMap stateMap1 = stateManager.getState(Scope.LOCAL);
+
+ final Map<String, String> state2 = new HashMap<>();
+ state2.put("key", "value2");
+ stateManager.setState(state2, Scope.LOCAL);
+ final StateMap stateMap2 = stateManager.getState(Scope.LOCAL);
+
+ assertTrue(stateMap1.getStateVersion().isPresent());
+ assertTrue(stateMap2.getStateVersion().isPresent());
+
+ final long version1 = Long.parseLong(stateMap1.getStateVersion().get());
+ final long version2 = Long.parseLong(stateMap2.getStateVersion().get());
+ assertTrue(version2 > version1);
+ }
+}
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/pom.xml
new file mode 100644
index 0000000000..959ac55052
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/pom.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-integration-tests</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- NARs required for integration testing -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-server-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-connectors-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+
+ <!-- NARs for components referenced in the Connectors -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-shared-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-update-attribute-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-nar</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/libDir</outputDirectory>
+ <includeTypes>jar,nar</includeTypes>
+ <includeScope>compile</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
+
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/CreateConnectorIT.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/CreateConnectorIT.java
index 96806fca1d..dccbaaa705 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/CreateConnectorIT.java
@@ -14,23 +14,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connectors.tests;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.mock.connector.StandardConnectorTestRunner;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.junit.jupiter.api.Test;
-public interface ValidationContextFactory {
+import java.io.File;
+import java.io.IOException;
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+public class CreateConnectorIT {
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ @Test
+ public void testCreateStartAndStopGenerateAndUpdateConnector() throws IOException {
+ try (final ConnectorTestRunner testRunner = new StandardConnectorTestRunner.Builder()
+ .connectorClassName("org.apache.nifi.mock.connectors.GenerateAndLog")
+ .narLibraryDirectory(new File("target/libDir"))
+ .build()) {
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ testRunner.startConnector();
+ testRunner.stopConnector();
+ }
+ }
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/MockProcessorIT.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/MockProcessorIT.java
new file mode 100644
index 0000000000..465d034000
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/MockProcessorIT.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connectors.tests;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.mock.connector.StandardConnectorTestRunner;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MockProcessorIT {
+
+ @Test
+ @Timeout(10)
+ public void testMockProcessor() throws IOException {
+ try (final ConnectorTestRunner runner = new StandardConnectorTestRunner.Builder()
+ .narLibraryDirectory(new File("target/libDir"))
+ .connectorClassName("org.apache.nifi.mock.connectors.GenerateAndLog")
+ .mockProcessor("org.apache.nifi.processors.attributes.UpdateAttribute", MockProcessor.class)
+ .build()) {
+
+ runner.startConnector();
+
+ // Wait until MockProcessor is triggered at least once. We use @Timeout on the test to avoid
+ // hanging indefinitely in case of failure.
+ while (MockProcessor.invocationCounter.get() < 1) {
+ Thread.yield();
+ }
+ runner.stopConnector();
+ }
+ }
+
+ public static class MockProcessor extends AbstractProcessor {
+ private static final AtomicLong invocationCounter = new AtomicLong(0L);
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .build();
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return Set.of(REL_SUCCESS);
+ }
+
+ // Support any properties so that properties of UpdateAttribute can be set without the Processor becoming invalid
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .name(propertyDescriptorName)
+ .addValidator(Validator.VALID)
+ .build();
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ invocationCounter.incrementAndGet();
+
+ final FlowFile flowFile = session.get();
+ if (flowFile != null) {
+ session.transfer(flowFile, REL_SUCCESS);
+ }
+ }
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors-nar/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors-nar/pom.xml
new file mode 100644
index 0000000000..89d3a10121
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors-nar/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-test-connectors-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-connectors</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/pom.xml
new file mode 100644
index 0000000000..9f2a86cd96
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-test-connectors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java
new file mode 100644
index 0000000000..617a1ca055
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connectors;
+
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.connector.AbstractConnector;
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.components.connector.components.FlowContext;
+import org.apache.nifi.components.connector.util.VersionedFlowUtils;
+import org.apache.nifi.flow.VersionedExternalFlow;
+
+import java.util.List;
+
+public class GenerateAndLog extends AbstractConnector {
+
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json");
+ }
+
+ @Override
+ protected void onStepConfigured(final String stepName, final FlowContext flowContext) {
+ }
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps(final FlowContext flowContext) {
+ return List.of();
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException {
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final String stepName, final List<PropertyGroupConfiguration> overrides, final FlowContext workingContext) {
+ return List.of();
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
index 56740ff7c3..c6cbc30195 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.mock.connectors.GenerateAndLog
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Generate_and_Update.json b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Generate_and_Update.json
new file mode 100644
index 0000000000..686247011e
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Generate_and_Update.json
@@ -0,0 +1 @@
+{"flowContents":{"identifier":"cdf7f935-0045-3c88-bdad-fd4e34105756","instanceIdentifier":"3a87b58f-019a-1000-30d2-87dfdd9816aa","name":"Generate and Update","comments":"","position":{"x":-1124.0,"y":65.5},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"4d4160b1-736c-3be4-bc3a-84fc6eb4ad2a","instanceIdentifier":"3a87db5f-019a-1000-75df-90e055f21fc9","name":"UpdateAttribute","comments":"","position":{"x":-358.0,"y":-106.5},"type":"org.apache.nifi.processors.attrib [...]
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/pom.xml
new file mode 100644
index 0000000000..c7b734eb41
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/pom.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-connector-mock-test-connectors</module>
+ <module>nifi-connector-mock-test-connectors-nar</module>
+ <module>nifi-connector-mock-integration-tests</module>
+ </modules>
+</project>
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock/pom.xml
new file mode 100644
index 0000000000..3f6ea8b8cd
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock/pom.xml
@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-properties</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Dependencies necessary to bootstrap NiFi -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+
+ <!-- Core NiFi API dependencies -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-python-framework-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-stateless-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Logging dependencies to ensure that everything funnels through SLF4J -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- NARs required -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-jetty-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-provenance-repository-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-nar</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/libDir</outputDirectory>
+ <includeTypes>jar,nar</includeTypes>
+ <includeScope>compile</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/StandardConnectorTestRunner.java b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/StandardConnectorTestRunner.java
new file mode 100644
index 0000000000..907204d0f0
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/StandardConnectorTestRunner.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector;
+
+import org.apache.nifi.NiFiServer;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.mock.connector.server.ConnectorConfigVerificationResult;
+import org.apache.nifi.mock.connector.server.ConnectorMockServer;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.apache.nifi.nar.ExtensionMapping;
+import org.apache.nifi.nar.NarClassLoaders;
+import org.apache.nifi.nar.NarUnpackMode;
+import org.apache.nifi.nar.NarUnpacker;
+import org.apache.nifi.nar.SystemBundle;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.util.NiFiProperties;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class StandardConnectorTestRunner implements ConnectorTestRunner, Closeable {
+ private final File narLibraryDirectory;
+
+ private ConnectorMockServer mockServer;
+
+ private StandardConnectorTestRunner(final Builder builder) {
+ this.narLibraryDirectory = builder.narLibraryDirectory;
+
+ try {
+ bootstrapInstance();
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to bootstrap ConnectorTestRunner", e);
+ }
+
+ // It is important that we register the processor mocks before instantiating the connector.
+ // Otherwise, the call to instantiateConnector will initialize the Connector, which may update the flow.
+ // If the flow is updated before the processor mocks are registered, the Processors will be created without
+ // using the mocks. Subsequent updates to the flow will not replace the Processors already created because
+ // these are not recognized as updates to the flow, since the framework assumes that the type of a Processor
+ // with a given ID does not change.
+ builder.processorMocks.forEach(mockServer::mockProcessor);
+
+ mockServer.instantiateConnector(builder.connectorClassName);
+ }
+
+ private void bootstrapInstance() throws IOException, ClassNotFoundException {
+ final List<Path> libDirectoryPaths = List.of(narLibraryDirectory.toPath());
+ final File extensionsWorkingDir = new File("target/work/extensions");
+ final File frameworkWorkingDir = new File("target/work/framework");
+
+ final Bundle systemBundle = SystemBundle.create(narLibraryDirectory.getAbsolutePath(), ClassLoader.getSystemClassLoader());
+
+ final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(systemBundle, frameworkWorkingDir, extensionsWorkingDir, libDirectoryPaths, true,
+ NarClassLoaders.FRAMEWORK_NAR_ID, true, false, NarUnpackMode.UNPACK_INDIVIDUAL_JARS, bundleCoordinate -> true);
+
+ final NarClassLoaders narClassLoaders = new NarClassLoaders();
+ narClassLoaders.init(frameworkWorkingDir, extensionsWorkingDir);
+
+ final NiFiServer nifiServer = narClassLoaders.getServer();
+ if (nifiServer == null) {
+ throw new RuntimeException("Could not find NiFiServer instance");
+ }
+ if (!(nifiServer instanceof ConnectorMockServer)) {
+ throw new RuntimeException("Test ClassPath does not contain ConnectorMockServer. " +
+ "Ensure that the appropriate module is packaged in the NAR library directory: " + narLibraryDirectory.getAbsolutePath());
+ }
+
+ // Set Application Server Class Loader for subsequent operations
+ final ClassLoader applicationServerClassLoader = narClassLoaders.getServer().getClass().getClassLoader();
+ Thread.currentThread().setContextClassLoader(applicationServerClassLoader);
+
+ final Set<Bundle> narBundles = narClassLoaders.getBundles();
+
+ final NiFiProperties properties;
+ try (final InputStream propertiesIn = getClass().getClassLoader().getResourceAsStream("nifi.properties")) {
+ properties = NiFiProperties.createBasicNiFiProperties(propertiesIn);
+ }
+
+ nifiServer.initialize(properties, systemBundle, narBundles, extensionMapping);
+ nifiServer.start();
+
+ mockServer = (ConnectorMockServer) nifiServer;
+ mockServer.registerMockBundle(getClass().getClassLoader(), new File(extensionsWorkingDir, "mock-implementations-bundle"));
+ }
+
+ @Override
+ public void close() {
+ if (mockServer != null) {
+ mockServer.stop();
+ }
+ }
+
+ @Override
+ public void applyUpdate() throws FlowUpdateException {
+ mockServer.applyUpdate();
+ }
+
+ @Override
+ public void configure(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) throws FlowUpdateException {
+ mockServer.configure(stepName, groupConfigurations);
+ }
+
+ @Override
+ public ConnectorConfigVerificationResult verifyConfiguration(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) {
+ return mockServer.verifyConfiguration(stepName, groupConfigurations);
+ }
+
+ @Override
+ public void startConnector() {
+ mockServer.startConnector();
+ }
+
+ @Override
+ public void stopConnector() {
+ mockServer.stopConnector();
+ }
+
+ @Override
+ public void waitForDataIngested(final Duration maxWaitTime) {
+ mockServer.waitForDataIngested(maxWaitTime);
+ }
+
+ @Override
+ public void waitForIdle(final Duration maxWaitTime) {
+ mockServer.waitForIdle(maxWaitTime);
+ }
+
+ @Override
+ public void waitForIdle(final Duration minIdleTime, final Duration maxWaitTime) {
+ mockServer.waitForIdle(minIdleTime, maxWaitTime);
+ }
+
+ @Override
+ public List<ValidationResult> validate() {
+ return mockServer.validate();
+ }
+
+
+ public static class Builder {
+ private String connectorClassName;
+ private File narLibraryDirectory;
+ private final Map<String, Class<? extends Processor>> processorMocks = new HashMap<>();
+
+ public Builder connectorClassName(final String connectorClassName) {
+ this.connectorClassName = connectorClassName;
+ return this;
+ }
+
+ public Builder narLibraryDirectory(final File libDirectory) {
+ this.narLibraryDirectory = libDirectory;
+ return this;
+ }
+
+ public Builder mockProcessor(final String processorType, final Class<? extends Processor> mockProcessorClass) {
+ processorMocks.put(processorType, mockProcessorClass);
+ return this;
+ }
+
+ public StandardConnectorTestRunner build() {
+ if (!narLibraryDirectory.exists() || !narLibraryDirectory.isDirectory()) {
+ throw new IllegalArgumentException("NAR file does not exist or is not a directory: " + narLibraryDirectory.getAbsolutePath());
+ }
+
+ return new StandardConnectorTestRunner(this);
+ }
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties
new file mode 100644
index 0000000000..046a2980b2
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties
@@ -0,0 +1,159 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Core Properties #
+nifi.flow.configuration.file=target/nifi-storage/conf/flow.json.gz
+nifi.flow.configuration.archive.enabled=true
+nifi.flow.configuration.archive.dir=target/nifi-storage/conf/archive/
+nifi.flow.configuration.archive.max.time=30 days
+nifi.flow.configuration.archive.max.storage=500 MB
+nifi.flow.configuration.archive.max.count=
+nifi.flowcontroller.autoResumeState=true
+nifi.flowcontroller.graceful.shutdown.period=10 sec
+nifi.flowservice.writedelay.interval=500 ms
+nifi.administrative.yield.duration=3 sec
+# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
+nifi.bored.yield.duration=10 millis
+nifi.queue.backpressure.count=10000
+nifi.queue.backpressure.size=1 GB
+
+nifi.nar.library.directory=./lib
+nifi.nar.library.autoload.directory=./extensions
+nifi.nar.working.directory=./work/nar/
+
+#####################
+# Python Extensions #
+#####################
+# Uncomment in order to enable Python Extensions.
+#nifi.python.command=python
+nifi.python.framework.source.directory=./python/framework
+nifi.python.extensions.source.directory.default=./python/extensions
+nifi.python.working.directory=./work/python
+nifi.python.max.processes=100
+nifi.python.max.processes.per.extension.type=10
+
+####################
+# State Management #
+####################
+nifi.state.management.configuration.file=target/nifi-storage/state-management.xml
+# The ID of the local state provider
+nifi.state.management.provider.local=local-provider
+
+# Database Settings
+nifi.database.directory=target/nifi-storage/database_repository
+
+# FlowFile Repository
+nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
+nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog
+nifi.flowfile.repository.directory=target/nifi-storage/flowfile_repository
+nifi.flowfile.repository.checkpoint.interval=1 mins
+nifi.flowfile.repository.always.sync=false
+
+nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
+nifi.queue.swap.threshold=20000
+nifi.swap.in.period=5 sec
+nifi.swap.in.threads=1
+nifi.swap.out.period=5 sec
+nifi.swap.out.threads=4
+
+# Content Repository
+nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
+nifi.content.claim.max.appendable.size=50 KB
+nifi.content.repository.directory.default=target/nifi-storage/content_repository
+nifi.content.repository.archive.max.retention.period=12 hours
+nifi.content.repository.archive.max.usage.percentage=90%
+nifi.content.repository.archive.enabled=false
+nifi.content.repository.always.sync=false
+
+# Provenance Repository Properties
+nifi.provenance.repository.implementation=org.apache.nifi.provenance.VolatileProvenanceRepository
+
+# Volatile Provenance Respository Properties
+nifi.provenance.repository.buffer.size=1000
+
+# Component Status Repository
+nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
+nifi.components.status.repository.buffer.size=1440
+nifi.components.status.snapshot.frequency=1 min
+
+# NAR Persistence Provider Properties
+nifi.nar.persistence.provider.properties.directory=target/nifi-storage/nar_repository
+
+# Asset Management
+nifi.asset.manager.properties.directory=target/nifi-storage/assets
+
+# Site to Site properties
+nifi.remote.input.host=localhost
+nifi.remote.input.secure=true
+nifi.remote.input.socket.port=7780
+nifi.remote.input.http.enabled=true
+nifi.remote.input.http.transaction.ttl=30 sec
+nifi.remote.contents.cache.expiration=30 secs
+
+# web properties #
+nifi.web.war.directory=./lib
+nifi.web.http.host=
+nifi.web.http.port=
+nifi.web.http.network.interface.default=
+nifi.web.https.host=localhost
+nifi.web.https.port=0
+nifi.web.https.network.interface.default=
+nifi.web.jetty.working.directory=target/work/jetty
+nifi.web.jetty.threads=200
+nifi.web.max.header.size=16 KB
+nifi.web.proxy.context.path=
+nifi.web.proxy.host=
+
+# security properties #
+nifi.sensitive.props.key=nifi-connector-mock-tests
+nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
+
+nifi.security.keystore=
+nifi.security.keystoreType=PKCS12
+nifi.security.keystorePasswd=
+nifi.security.keyPasswd=
+nifi.security.truststore=
+nifi.security.truststoreType=PKCS12
+nifi.security.truststorePasswd=
+nifi.security.user.authorizer=system-test-authorizer
+nifi.security.user.login.identity.provider=
+
+# cluster common properties (all nodes must have same values) #
+nifi.cluster.protocol.heartbeat.interval=5 sec
+nifi.cluster.protocol.is.secure=true
+
+# cluster node properties (only configure for cluster nodes) #
+nifi.cluster.is.node=false
+nifi.cluster.node.address=
+nifi.cluster.node.protocol.port=
+nifi.cluster.node.protocol.threads=10
+nifi.cluster.node.protocol.max.threads=50
+nifi.cluster.node.event.history.size=25
+nifi.cluster.node.connection.timeout=5 sec
+nifi.cluster.node.read.timeout=5 sec
+nifi.cluster.node.max.concurrent.requests=100
+nifi.cluster.firewall.file=
+nifi.cluster.flow.election.max.wait.time=5 mins
+nifi.cluster.flow.election.max.candidates=
+
+# cluster load balancing properties #
+nifi.cluster.load.balance.host=
+nifi.cluster.load.balance.port=6342
+nifi.cluster.load.balance.connections.per.node=4
+nifi.cluster.load.balance.max.thread.count=8
+nifi.cluster.load.balance.comms.timeout=30 sec
+
+# Connector Repository
+nifi.components.connectors.repository.implementation=org.apache.nifi.mock.connector.server.MockConnectorRepository
diff --git a/nifi-connector-mock-bundle/pom.xml b/nifi-connector-mock-bundle/pom.xml
new file mode 100644
index 0000000000..adb016336e
--- /dev/null
+++ b/nifi-connector-mock-bundle/pom.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-connector-mock</module>
+ <module>nifi-connector-mock-api</module>
+ <module>nifi-connector-mock-server</module>
+ <module>nifi-connector-mock-server-nar</module>
+ <module>nifi-connector-mock-test-bundle</module>
+ </modules>
+</project>
\ No newline at end of file
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/pom.xml
new file mode 100644
index 0000000000..7f95903475
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-connector</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-utils</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
new file mode 100644
index 0000000000..793b1e9be0
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorPropertyDescriptor;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.PropertyType;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.List;
+
+public class KafkaConnectionStep {
+ public static final String STEP_NAME = "Kafka Connection";
+
+ public static final ConnectorPropertyDescriptor KAFKA_BROKERS = new ConnectorPropertyDescriptor.Builder()
+ .name("Kafka Brokers")
+ .description("A comma-separated list of Kafka brokers to connect to.")
+ .required(true)
+ .validators(StandardValidators.HOSTNAME_PORT_LIST_VALIDATOR)
+ .type(PropertyType.STRING_LIST)
+ .build();
+
+ public static final ConnectorPropertyDescriptor SECURITY_PROTOCOL = new ConnectorPropertyDescriptor.Builder()
+ .name("Security Protocol")
+ .description("The security protocol to use when connecting to Kafka brokers.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .defaultValue("SASL_PLAINTEXT")
+ .allowableValues("PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL")
+ .build();
+
+ public static final ConnectorPropertyDescriptor SASL_MECHANISM = new ConnectorPropertyDescriptor.Builder()
+ .name("SASL Mechanism")
+ .description("The SASL mechanism to use for authentication.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .allowableValues("PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "GSSAPI", "OAUTHBEARER")
+ .dependsOn(SECURITY_PROTOCOL, "SASL_PLAINTEXT", "SASL_SSL")
+ .defaultValue("SCRAM-SHA-512")
+ .build();
+
+ public static final ConnectorPropertyDescriptor USERNAME = new ConnectorPropertyDescriptor.Builder()
+ .name("Username")
+ .description("The username for SASL authentication.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SECURITY_PROTOCOL, "SASL_PLAINTEXT", "SASL_SSL")
+ .build();
+
+ public static final ConnectorPropertyDescriptor PASSWORD = new ConnectorPropertyDescriptor.Builder()
+ .name("Password")
+ .description("The password for SASL authentication.")
+ .required(true)
+ .type(PropertyType.SECRET)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SECURITY_PROTOCOL, "SASL_PLAINTEXT", "SASL_SSL")
+ .build();
+
+ public static final ConnectorPropertyGroup KAFKA_SERVER_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Kafka Server Settings")
+ .description("Settings for connecting to the Kafka server")
+ .properties(List.of(
+ KAFKA_BROKERS,
+ SECURITY_PROTOCOL,
+ SASL_MECHANISM,
+ USERNAME,
+ PASSWORD
+ ))
+ .build();
+
+ public static final ConnectorPropertyDescriptor SCHEMA_REGISTRY_URL = new ConnectorPropertyDescriptor.Builder()
+ .name("Schema Registry URL")
+ .description("The URL of the Schema Registry.")
+ .required(false)
+ .type(PropertyType.STRING)
+ .validators(StandardValidators.URL_VALIDATOR)
+ .validators(Validator.VALID)
+ .build();
+
+ public static final ConnectorPropertyDescriptor SCHEMA_REGISTRY_USERNAME = new ConnectorPropertyDescriptor.Builder()
+ .name("Schema Registry Username")
+ .description("The username for Schema Registry authentication.")
+ .required(false)
+ .type(PropertyType.STRING)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SCHEMA_REGISTRY_URL)
+ .build();
+
+ public static final ConnectorPropertyDescriptor SCHEMA_REGISTRY_PASSWORD = new ConnectorPropertyDescriptor.Builder()
+ .name("Schema Registry Password")
+ .description("The password for Schema Registry authentication.")
+ .required(false)
+ .type(PropertyType.SECRET)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SCHEMA_REGISTRY_URL)
+ .build();
+
+
+ public static final ConnectorPropertyGroup SCHEMA_REGISTRY_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Schema Registry Settings")
+ .description("Settings for connecting to the Schema Registry")
+ .properties(List.of(
+ SCHEMA_REGISTRY_URL,
+ SCHEMA_REGISTRY_USERNAME,
+ SCHEMA_REGISTRY_PASSWORD
+ ))
+ .build();
+
+ public static final ConfigurationStep KAFKA_CONNECTION_STEP = new ConfigurationStep.Builder()
+ .name(STEP_NAME)
+ .description("Configure Kafka connection settings")
+ .propertyGroups(List.of(
+ KAFKA_SERVER_GROUP,
+ SCHEMA_REGISTRY_GROUP
+ ))
+ .build();
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java
new file mode 100644
index 0000000000..c94226e132
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
+import org.apache.nifi.components.connector.AbstractConnector;
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorConfigurationContext;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.InvocationFailedException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.components.connector.components.ControllerServiceFacade;
+import org.apache.nifi.components.connector.components.FlowContext;
+import org.apache.nifi.components.connector.components.ProcessGroupFacade;
+import org.apache.nifi.components.connector.components.ProcessorFacade;
+import org.apache.nifi.components.connector.util.VersionedFlowUtils;
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.flow.VersionedExternalFlow;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+@CapabilityDescription("Provides the ability to ingest data from Apache Kafka topics, merge it together into an object of reasonable " +
+ "size, and write that data to Amazon S3.")
+@Tags({"kafka", "s3"})
+public class KafkaToS3 extends AbstractConnector {
+
+ private volatile CompletableFuture<Void> drainFlowFileFuture = null;
+
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps(final FlowContext flowContext) {
+ return List.of(
+ KafkaConnectionStep.KAFKA_CONNECTION_STEP,
+ KafkaTopicsStep.KAFKA_TOPICS_STEP,
+ S3Step.S3_STEP
+ );
+ }
+
+ @Override
+ public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException {
+ final String activeS3DataFormat = activeContext.getConfigurationContext().getProperty(
+ S3Step.S3_STEP_NAME, S3Step.S3_DESTINATION_GROUP.getName(), S3Step.S3_DATA_FORMAT.getName()).getValue();
+ final String workingS3DataFormat = workingContext.getConfigurationContext().getProperty(
+ S3Step.S3_STEP_NAME, S3Step.S3_DESTINATION_GROUP.getName(), S3Step.S3_DATA_FORMAT.getName()).getValue();
+
+ if (!activeS3DataFormat.equals(workingS3DataFormat)) {
+ getLogger().info("S3 Data Format changed from {} to {}; draining flow before updating it", activeS3DataFormat, workingS3DataFormat);
+
+ drainFlowFileFuture = drainFlowFiles(activeContext);
+ try {
+ drainFlowFileFuture.get(5, TimeUnit.MINUTES);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new FlowUpdateException("Interrupted while waiting for all FlowFiles to drain from the flow", e);
+ } catch (final TimeoutException e) {
+ throw new FlowUpdateException("Timed out waiting for all FlowFiles to drain from the flow", e);
+ } catch (final ExecutionException e) {
+ throw new FlowUpdateException("Failed to drain FlowFiles from flow", e.getCause());
+ }
+
+ getLogger().info("All FlowFiles drained from the flow; proceeding with flow update");
+ }
+ }
+
+ @Override
+ public void abortUpdate(final FlowContext workingContext, final Throwable throwable) {
+ if (drainFlowFileFuture != null) {
+ drainFlowFileFuture.cancel(true);
+ drainFlowFileFuture = null;
+ }
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException {
+ final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext());
+ getInitializationContext().updateFlow(activeContext, flow);
+ }
+
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return KafkaToS3FlowBuilder.loadInitialFlow();
+ }
+
+ @Override
+ public void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException {
+ final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext());
+ getInitializationContext().updateFlow(workingContext, flow);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final String stepName, final List<PropertyGroupConfiguration> propertyOverrides, final FlowContext workingFlowContext) {
+ // Get the current ConfigurationContext and then create a new one that contains the provided property values
+ final ConnectorConfigurationContext configurationContext = workingFlowContext.getConfigurationContext().createWithOverrides(stepName, propertyOverrides);
+ final VersionedExternalFlow flow = buildFlow(configurationContext);
+
+ // Validate Connectivity
+ if (stepName.equals(KafkaConnectionStep.STEP_NAME)) {
+ return verifyKafkaConnectivity(workingFlowContext, flow);
+ }
+ if (stepName.equals(KafkaTopicsStep.STEP_NAME)) {
+ final List<ConfigVerificationResult> results = new ArrayList<>();
+ results.addAll(verifyTopicsExists(workingFlowContext));
+ results.addAll(verifyKafkaParsability(workingFlowContext, flow));
+ return results;
+ }
+
+ return Collections.emptyList();
+ }
+
+ private List<ConfigVerificationResult> verifyKafkaParsability(final FlowContext workingFlowContext, final VersionedExternalFlow flow) {
+ // Enable Controller Services necessary for parsing records.
+ // We determine which Controller Services are referenced by the flow and enable them, but we do not use
+ // getRootGroup().getLifecycle().enableReferencedControllerServices(ControllerServiceReferenceScope.INCLUDE_REFERENCED_SERVICES_ONLY)
+ // because that would include the Controller Services that are referenced by the currently configured flow, and it's possible that the
+ // what is being verified uses a different set of Controller Services (e.g., the verified flow may use a JSON Reader while the current
+ // flow uses an Avro Reader).
+ final ProcessGroupFacade rootGroup = workingFlowContext.getRootGroup();
+ final Set<VersionedControllerService> referencedServices = VersionedFlowUtils.getReferencedControllerServices(flow.getFlowContents());
+ final Set<String> serviceIds = referencedServices.stream()
+ .map(VersionedControllerService::getIdentifier)
+ .collect(Collectors.toSet());
+
+ try {
+ rootGroup.getLifecycle().enableControllerServices(serviceIds).get(10, TimeUnit.SECONDS);
+ } catch (final Exception e) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Record Parsing")
+ .outcome(Outcome.FAILED)
+ .explanation("Failed to enable Controller Services due to " + e)
+ .build());
+ }
+
+ try {
+ final ProcessorFacade consumeKafkaFacade = findProcessors(rootGroup,
+ processor -> processor.getDefinition().getType().endsWith("ConsumeKafka")).getFirst();
+
+ final List<ConfigVerificationResult> configVerificationResults = consumeKafkaFacade.verify(flow, Map.of());
+ for (final ConfigVerificationResult result : configVerificationResults) {
+ if (result.getOutcome() == Outcome.FAILED) {
+ return List.of(result);
+ }
+ }
+
+ return Collections.emptyList();
+ } finally {
+ rootGroup.getLifecycle().disableControllerServices(serviceIds);
+ }
+ }
+
+
+ private List<ConfigVerificationResult> verifyTopicsExists(final FlowContext workingFlowContext) {
+ final List<String> topicsAvailable;
+ try {
+ topicsAvailable = getAvailableTopics(workingFlowContext);
+ } catch (final Exception e) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka topics exist")
+ .outcome(Outcome.SKIPPED)
+ .explanation("Unable to validate that topics exist due to " + e)
+ .build());
+ }
+
+ final Set<String> topicNames = new HashSet<>(topicsAvailable);
+ final List<String> specifiedTopics = workingFlowContext.getConfigurationContext().getProperty(KafkaTopicsStep.STEP_NAME,
+ KafkaTopicsStep.KAFKA_TOPICS_GROUP.getName(), KafkaTopicsStep.TOPIC_NAMES.getName()).asList();
+ final String missingTopics = specifiedTopics.stream()
+ .filter(topic -> !topicNames.contains(topic))
+ .collect(Collectors.joining(", "));
+
+ if (!missingTopics.isEmpty()) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka topics exist")
+ .outcome(Outcome.FAILED)
+ .explanation("The following topics do not exist in the Kafka cluster: " + missingTopics)
+ .build());
+ } else {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka topics exist")
+ .outcome(Outcome.SUCCESSFUL)
+ .explanation("All specified topics exist in the Kafka cluster")
+ .build());
+ }
+ }
+
+ private List<ConfigVerificationResult> verifyKafkaConnectivity(final FlowContext workingFlowContext, final VersionedExternalFlow flow) {
+ // Build a new version of the flow so that we can get the relevant properties of the Kafka Connection Service
+ final ControllerServiceFacade connectionService = getKafkaConnectionService(workingFlowContext);
+ final List<ConfigVerificationResult> configVerificationResults = connectionService.verify(flow, Map.of());
+
+ for (final ConfigVerificationResult result : configVerificationResults) {
+ if (result.getOutcome() == Outcome.FAILED) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka connectivity")
+ .outcome(Outcome.FAILED)
+ .explanation(result.getExplanation())
+ .build());
+ }
+ }
+
+ return Collections.emptyList();
+ }
+
+ private VersionedExternalFlow buildFlow(final ConnectorConfigurationContext configurationContext) {
+ final KafkaToS3FlowBuilder flowBuilder = new KafkaToS3FlowBuilder(configurationContext);
+ return flowBuilder.buildFlow();
+ }
+
+ @Override
+ public List<AllowableValue> fetchAllowableValues(final String stepName, final String groupName, final String propertyName, final FlowContext flowContext) {
+ if (stepName.equals(KafkaTopicsStep.STEP_NAME) && propertyName.equals(KafkaTopicsStep.TOPIC_NAMES.getName())) {
+ return createAllowableValues(getAvailableTopics(flowContext));
+ } else if (stepName.equals(S3Step.S3_STEP_NAME) && propertyName.equals(S3Step.S3_REGION.getName())) {
+ return createAllowableValues(getPossibleS3Regions(flowContext));
+ }
+
+ return super.fetchAllowableValues(stepName, groupName, propertyName, flowContext);
+ }
+
+ private List<AllowableValue> createAllowableValues(final List<String> values) {
+ return values.stream().map(this::createAllowableValue).collect(Collectors.toList());
+ }
+
+ private AllowableValue createAllowableValue(final String value) {
+ return new AllowableValue(value, value, value);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> getAvailableTopics(final FlowContext flowContext) {
+ // If Kafka Brokers not yet set, return empty list
+ final ConnectorConfigurationContext config = flowContext.getConfigurationContext();
+ if (!config.getProperty(KafkaConnectionStep.KAFKA_CONNECTION_STEP, KafkaConnectionStep.KAFKA_SERVER_GROUP, KafkaConnectionStep.KAFKA_BROKERS).isSet()) {
+ return List.of();
+ }
+
+ final ControllerServiceFacade kafkaConnectionService = getKafkaConnectionService(flowContext);
+
+ try {
+ return (List<String>) kafkaConnectionService.invokeConnectorMethod("listTopicNames", Map.of());
+ } catch (final Exception e) {
+ getLogger().warn("Failed to retrieve available Kafka topics", e);
+ return List.of();
+ }
+ }
+
+ private ControllerServiceFacade getKafkaConnectionService(final FlowContext flowContext) {
+ return flowContext.getRootGroup().getControllerServices().stream()
+ .filter(service -> service.getDefinition().getType().endsWith("Kafka3ConnectionService"))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> getPossibleS3Regions(final FlowContext flowContext) {
+ final ProcessorFacade processorFacade = flowContext.getRootGroup().getProcessors().stream()
+ .filter(proc -> proc.getDefinition().getType().endsWith("PutS3Object"))
+ .findFirst()
+ .orElseThrow();
+
+ try {
+ return (List<String>) processorFacade.invokeConnectorMethod("getAvailableRegions", Map.of());
+ } catch (final InvocationFailedException e) {
+ getLogger().error("Failed to obtain list of available S3 regions", e);
+ return Collections.emptyList();
+ }
+ }
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3FlowBuilder.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3FlowBuilder.java
new file mode 100644
index 0000000000..88b22df5e3
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3FlowBuilder.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorConfigurationContext;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.util.VersionedFlowUtils;
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.flow.VersionedExternalFlow;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.processor.DataUnit;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaToS3FlowBuilder {
+ private static final String FLOW_JSON_PATH = "flows/Kafka_to_S3.json";
+
+ private final ConnectorConfigurationContext configContext;
+
+ public KafkaToS3FlowBuilder(final ConnectorConfigurationContext configurationContext) {
+ this.configContext = configurationContext;
+ }
+
+ public static VersionedExternalFlow loadInitialFlow() {
+ return VersionedFlowUtils.loadFlowFromResource(FLOW_JSON_PATH);
+ }
+
+ public VersionedExternalFlow buildFlow() {
+ final VersionedExternalFlow externalFlow = VersionedFlowUtils.loadFlowFromResource(FLOW_JSON_PATH);
+ configureSchemaRegistry(externalFlow);
+
+ updateKafkaConnectionParameters(externalFlow);
+ updateSchemaRegistryParameters(externalFlow);
+ updateReaderWriter(externalFlow);
+ updateKafkaTopicsParameters(externalFlow);
+ updateS3Config(externalFlow);
+
+ return externalFlow;
+ }
+
+ private void configureSchemaRegistry(final VersionedExternalFlow externalFlow) {
+ final String schemaRegistryUrl = configContext.getProperty(KafkaConnectionStep.KAFKA_CONNECTION_STEP,
+ KafkaConnectionStep.SCHEMA_REGISTRY_GROUP,
+ KafkaConnectionStep.SCHEMA_REGISTRY_URL).getValue();
+
+ if (schemaRegistryUrl == null) {
+ final VersionedProcessGroup processGroup = externalFlow.getFlowContents();
+
+ // Remove any references to the Schema Registry service.
+ final VersionedControllerService schemaRegistryService = processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("ConfluentSchemaRegistry"))
+ .findFirst()
+ .orElseThrow();
+ VersionedFlowUtils.removeControllerServiceReferences(processGroup, schemaRegistryService.getIdentifier());
+
+ final VersionedControllerService schemaReferenceReader = processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("ConfluentEncodedSchemaReferenceReader"))
+ .findFirst()
+ .orElseThrow();
+ VersionedFlowUtils.removeControllerServiceReferences(processGroup, schemaReferenceReader.getIdentifier());
+
+ processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("JsonTreeReader"))
+ .forEach(service -> service.getProperties().put("Schema Access Strategy", "infer-schema"));
+
+ processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("JsonRecordSetWriter"))
+ .forEach(service -> service.getProperties().put("Schema Write Strategy", "no-schema"));
+ }
+ }
+
+ private void updateSchemaRegistryParameters(final VersionedExternalFlow externalFlow) {
+ final ConfigurationStep connectionStep = KafkaConnectionStep.KAFKA_CONNECTION_STEP;
+ final ConnectorPropertyGroup registryGroup = KafkaConnectionStep.SCHEMA_REGISTRY_GROUP;
+
+ final String schemaRegistryUrl = configContext.getProperty(connectionStep, registryGroup, KafkaConnectionStep.SCHEMA_REGISTRY_URL).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry URLs", schemaRegistryUrl);
+
+ if (schemaRegistryUrl == null) {
+ final Map<String, String> properties = new HashMap<>();
+ properties.put("schema-access-strategy", "infer-schema");
+ properties.put("schema-registry", null);
+ properties.put("schema-reference-reader", null);
+
+ externalFlow.getFlowContents().getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("JsonTreeReader"))
+ .findFirst()
+ .ifPresent(service -> service.setProperties(properties));
+ } else {
+ final String username = configContext.getProperty(connectionStep, registryGroup, KafkaConnectionStep.SCHEMA_REGISTRY_USERNAME).getValue();
+ final String password = configContext.getProperty(connectionStep, registryGroup, KafkaConnectionStep.SCHEMA_REGISTRY_PASSWORD).getValue();
+
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry Username", username);
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry Password", password);
+
+ final String authenticationType = (username == null || username.isEmpty()) ? "NONE" : "BASIC";
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry Authentication Type", authenticationType);
+ }
+ }
+
+ private void updateKafkaConnectionParameters(final VersionedExternalFlow externalFlow) {
+ final ConfigurationStep connectionStep = KafkaConnectionStep.KAFKA_CONNECTION_STEP;
+ final ConnectorPropertyGroup serverGroup = KafkaConnectionStep.KAFKA_SERVER_GROUP;
+
+ final String kafkaBrokers = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.KAFKA_BROKERS).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Bootstrap Servers", kafkaBrokers);
+
+ final String securityProtocol = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.SECURITY_PROTOCOL).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Security Protocol", securityProtocol);
+
+ if (securityProtocol.contains("SASL")) {
+ final String saslMechanism = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.SASL_MECHANISM).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka SASL Mechanism", saslMechanism);
+
+ final String username = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.USERNAME).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka SASL Username", username);
+
+ final String password = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.PASSWORD).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka SASL Password", password);
+ }
+ }
+
+ private void updateReaderWriter(final VersionedExternalFlow externalFlow) {
+ final VersionedProcessGroup rootGroup = externalFlow.getFlowContents();
+
+ final VersionedControllerService avroService = rootGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("AvroReader"))
+ .findFirst()
+ .orElseThrow();
+
+ final String kafkaDataFormat = configContext.getProperty(KafkaTopicsStep.STEP_NAME, KafkaTopicsStep.KAFKA_TOPICS_GROUP.getName(), KafkaTopicsStep.KAFKA_DATA_FORMAT.getName()).getValue();
+ if (!kafkaDataFormat.equalsIgnoreCase("JSON")) {
+ // Update ConsumeKafka processor to use Avro Reader
+ rootGroup.getProcessors().stream()
+ .filter(versionedProcessor -> versionedProcessor.getType().endsWith("ConsumeKafka"))
+ .findFirst()
+ .ifPresent(processor -> processor.getProperties().put("Record Reader", avroService.getIdentifier()));
+ }
+
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Data Format", kafkaDataFormat);
+
+ // TODO: Have this throw an Exception if GROUP does not contain the given property
+ final String s3DataFormat = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_DATA_FORMAT).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Data Format", s3DataFormat);
+ }
+
+ private void updateKafkaTopicsParameters(final VersionedExternalFlow externalFlow) {
+ final String groupName = KafkaTopicsStep.KAFKA_TOPICS_GROUP.getName();
+
+ final String topics = configContext.getProperty(KafkaTopicsStep.STEP_NAME, groupName, KafkaTopicsStep.TOPIC_NAMES.getName()).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Topic Names", topics);
+
+ final String groupId = configContext.getProperty(KafkaTopicsStep.STEP_NAME, groupName, KafkaTopicsStep.CONSUMER_GROUP_ID.getName()).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Consumer Group ID", groupId);
+
+ final String offsetReset = configContext.getProperty(KafkaTopicsStep.STEP_NAME, groupName, KafkaTopicsStep.OFFSET_RESET.getName()).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Auto Offset Reset", offsetReset);
+ }
+
+ private void updateS3Config(final VersionedExternalFlow externalFlow) {
+ final String region = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_REGION).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Region", region);
+
+ final String bucket = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_BUCKET).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Bucket", bucket);
+
+ final String prefix = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_PREFIX).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Prefix", prefix);
+
+ final String endpointOverrideUrl = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_ENDPOINT_OVERRIDE_URL).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Endpoint Override URL", endpointOverrideUrl);
+
+ final String authStrategy = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_CREDENTIALS_GROUP, S3Step.S3_AUTHENTICATION_STRATEGY).getValue();
+ if (authStrategy.equals(S3Step.DEFAULT_CREDENTIALS)) {
+ final VersionedControllerService credentialsService = externalFlow.getFlowContents().getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("AWSCredentialsProviderControllerService"))
+ .findFirst()
+ .orElseThrow();
+
+ credentialsService.setProperties(Map.of("default-credentials", "true"));
+ } else {
+ final String accessKey = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_CREDENTIALS_GROUP, S3Step.S3_ACCESS_KEY_ID).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Access Key ID", accessKey);
+
+ final String secretKey = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_CREDENTIALS_GROUP, S3Step.S3_SECRET_ACCESS_KEY).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Secret Access Key", secretKey);
+ }
+
+ final long mergeBytes = configContext.getProperty(S3Step.S3_STEP, S3Step.MERGE_GROUP, S3Step.TARGET_OBJECT_SIZE).asDataSize(DataUnit.B).longValue();
+ final String mergeSize = configContext.getProperty(S3Step.S3_STEP, S3Step.MERGE_GROUP, S3Step.TARGET_OBJECT_SIZE).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Target Object Size", mergeSize);
+
+ // Max Bin size will be either 10% more than target size or target size + 100MB, whichever is smaller
+ final long maxBinSize = (long) Math.min(mergeBytes + 100_000_000, mergeBytes * 1.1D);
+ VersionedFlowUtils.setParameterValue(externalFlow, "Maximum Object Size", maxBinSize + " B");
+
+ final String mergeLatency = configContext.getProperty(S3Step.S3_STEP, S3Step.MERGE_GROUP, S3Step.MERGE_LATENCY).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Merge Latency", mergeLatency);
+ }
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaTopicsStep.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaTopicsStep.java
new file mode 100644
index 0000000000..643e4e836e
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaTopicsStep.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorPropertyDescriptor;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.PropertyType;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.List;
+
+public class KafkaTopicsStep {
+ public static final String STEP_NAME = "Kafka Topics";
+
+ public static final ConnectorPropertyDescriptor TOPIC_NAMES = new ConnectorPropertyDescriptor.Builder()
+ .name("Topic Names")
+ .description("A comma-separated list of Kafka topics to consume from.")
+ .required(true)
+ .type(PropertyType.STRING_LIST)
+ .allowableValuesFetchable(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor CONSUMER_GROUP_ID = new ConnectorPropertyDescriptor.Builder()
+ .name("Consumer Group ID")
+ .description("The consumer group ID to use when connecting to Kafka.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor OFFSET_RESET = new ConnectorPropertyDescriptor.Builder()
+ .name("Offset Reset")
+ .description("What to do when there is no initial offset in Kafka or if the current offset does not exist any more on the server.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .defaultValue("latest")
+ .allowableValues("earliest", "latest", "none")
+ .build();
+
+ public static final ConnectorPropertyDescriptor KAFKA_DATA_FORMAT = new ConnectorPropertyDescriptor.Builder()
+ .name("Kafka Data Format")
+ .description("The format of the data in Kafka topics.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .defaultValue("Avro")
+ .allowableValues("Avro", "JSON")
+ .build();
+
+ public static final ConnectorPropertyGroup KAFKA_TOPICS_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Kafka Topics Configuration")
+ .description("Properties for configuring Kafka topics consumption.")
+ .properties(List.of(
+ TOPIC_NAMES,
+ CONSUMER_GROUP_ID,
+ OFFSET_RESET,
+ KAFKA_DATA_FORMAT
+ ))
+ .build();
+
+ public static final ConfigurationStep KAFKA_TOPICS_STEP = new ConfigurationStep.Builder()
+ .name(STEP_NAME)
+ .description("Kafka topics to consume from." )
+ .propertyGroups(List.of(
+ KAFKA_TOPICS_GROUP
+ ))
+ .build();
+
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java
new file mode 100644
index 0000000000..ad29115c4a
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorPropertyDescriptor;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.PropertyType;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.List;
+
+public class S3Step {
+ public static final String S3_STEP_NAME = "S3 Configuration";
+ public static final String ACCESS_KEY_ID_SECRET_KEY = "Access Key ID and Secret Key";
+ public static final String DEFAULT_CREDENTIALS = "Default AWS Credentials";
+
+
+ public static final ConnectorPropertyDescriptor S3_BUCKET = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Bucket")
+ .description("The name of the S3 bucket to write data to.")
+ .required(true)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_REGION = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Region")
+ .description("The AWS region where the S3 bucket is located.")
+ .allowableValuesFetchable(true)
+ .required(true)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_DATA_FORMAT = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Data Format")
+ .description("The format to use when writing data to S3.")
+ .required(true)
+ .defaultValue("JSON")
+ .allowableValues("Avro", "JSON")
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_PREFIX = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Prefix")
+ .description("An optional prefix to prepend to all object keys written to the S3 bucket.")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_ENDPOINT_OVERRIDE_URL = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Endpoint Override URL")
+ .description("An optional endpoint URL to use instead of the default AWS S3 endpoint. " +
+ "This can be used to connect to S3-compatible storage systems but should be left unset for connecting to S3.")
+ .required(false)
+ .addValidator(StandardValidators.URL_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor TARGET_OBJECT_SIZE = new ConnectorPropertyDescriptor.Builder()
+ .name("Target Object Size")
+ .description("The target size for each object written to S3. The connector will attempt to " +
+ "combine messages until this size is reached before writing an object to S3. Note that this size is approximate " +
+ "and may vary from object to object.")
+ .required(true)
+ .defaultValue("256 MB")
+ .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor MERGE_LATENCY = new ConnectorPropertyDescriptor.Builder()
+ .name("Merge Latency")
+ .description("The maximum amount of time to wait while merging messages before writing an object to S3. " +
+ "If this time is reached before the target object size is met, the current set of merged messages " +
+ "will be written to S3.")
+ .required(true)
+ .defaultValue("5 min")
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_AUTHENTICATION_STRATEGY = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Authentication Strategy")
+ .description("The authentication strategy to use when connecting to S3.")
+ .required(true)
+ .defaultValue(ACCESS_KEY_ID_SECRET_KEY)
+ .allowableValues(ACCESS_KEY_ID_SECRET_KEY, DEFAULT_CREDENTIALS)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_ACCESS_KEY_ID = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Access Key ID")
+ .description("The AWS Access Key ID used to authenticate to S3.")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(S3_AUTHENTICATION_STRATEGY, ACCESS_KEY_ID_SECRET_KEY)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_SECRET_ACCESS_KEY = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Secret Access Key")
+ .description("The AWS Secret Access Key used to authenticate to S3.")
+ .required(true)
+ .type(PropertyType.SECRET)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(S3_AUTHENTICATION_STRATEGY, ACCESS_KEY_ID_SECRET_KEY)
+ .build();
+
+
+ public static final ConnectorPropertyGroup S3_DESTINATION_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("S3 Destination Configuration")
+ .description("Properties required to connect to S3 and specify the target bucket.")
+ .properties(List.of(
+ S3_BUCKET,
+ S3_PREFIX,
+ S3_REGION,
+ S3_DATA_FORMAT,
+ S3_ENDPOINT_OVERRIDE_URL
+ ))
+ .build();
+
+ public static final ConnectorPropertyGroup S3_CREDENTIALS_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("S3 Credentials")
+ .description("Properties required to authenticate to S3.")
+ .properties(List.of(
+ S3_AUTHENTICATION_STRATEGY,
+ S3_ACCESS_KEY_ID,
+ S3_SECRET_ACCESS_KEY
+ ))
+ .build();
+
+ public static final ConnectorPropertyGroup MERGE_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Merge Configuration")
+ .description("Configuration for how data should be merged together before being persisted to S3.")
+ .properties(List.of(
+ TARGET_OBJECT_SIZE,
+ MERGE_LATENCY
+ ))
+ .build();
+
+ public static final ConfigurationStep S3_STEP = new ConfigurationStep.Builder()
+ .name(S3_STEP_NAME)
+ .description("Configure connection to S3 and target bucket details.")
+ .propertyGroups(List.of(
+ S3_DESTINATION_GROUP,
+ MERGE_GROUP,
+ S3_CREDENTIALS_GROUP
+ ))
+ .build();
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
index 56740ff7c3..de5fe10a48 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.connectors.kafkas3.KafkaToS3
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/flows/Kafka_to_S3.json b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/flows/Kafka_to_S3.json
new file mode 100644
index 0000000000..5df6da47ae
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/flows/Kafka_to_S3.json
@@ -0,0 +1,2105 @@
+{
+ "flowContents": {
+ "identifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "instanceIdentifier": "2c825ae7-019a-1000-f3b7-8275175b7b5b",
+ "name": "Kafka to S3",
+ "comments": "",
+ "position": {
+ "x": -355.0,
+ "y": -238.5
+ },
+ "processGroups": [],
+ "remoteProcessGroups": [],
+ "processors": [
+ {
+ "identifier": "4ce9587d-a171-3dff-bea8-bc67f4b356cd",
+ "instanceIdentifier": "40e02539-8a55-3a44-53d9-e9d41f7db23a",
+ "name": "MergeRecord",
+ "comments": "",
+ "position": {
+ "x": -250.0,
+ "y": -55.5
+ },
+ "type": "org.apache.nifi.processors.standard.MergeRecord",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "max-bin-age": "#{Merge Latency}",
+ "correlation-attribute-name": "kafka.topic",
+ "Attribute Strategy": "Keep Only Common Attributes",
+ "min-records": "1",
+ "max-bin-size": "#{Maximum Object size}",
+ "record-writer": "2384a9ae-311e-3d5f-983b-13f82174244c",
+ "merge-strategy": "Bin-Packing Algorithm",
+ "record-reader": "cd697ee5-9621-3680-a6b8-af3237472b5b",
+ "max-records": "1000000000",
+ "min-bin-size": "#{Target Object Size}",
+ "max.bin.count": "10"
+ },
+ "propertyDescriptors": {
+ "max-bin-age": {
+ "name": "max-bin-age",
+ "displayName": "Max Bin Age",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "correlation-attribute-name": {
+ "name": "correlation-attribute-name",
+ "displayName": "Correlation Attribute Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Attribute Strategy": {
+ "name": "Attribute Strategy",
+ "displayName": "Attribute Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "min-records": {
+ "name": "min-records",
+ "displayName": "Minimum Number of Records",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max-bin-size": {
+ "name": "max-bin-size",
+ "displayName": "Maximum Bin Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "record-writer": {
+ "name": "record-writer",
+ "displayName": "Record Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "merge-strategy": {
+ "name": "merge-strategy",
+ "displayName": "Merge Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "record-reader": {
+ "name": "record-reader",
+ "displayName": "Record Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max-records": {
+ "name": "max-records",
+ "displayName": "Maximum Number of Records",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "min-bin-size": {
+ "name": "min-bin-size",
+ "displayName": "Minimum Bin Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max.bin.count": {
+ "name": "max.bin.count",
+ "displayName": "Maximum Number of Bins",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "style": {},
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "executionNode": "ALL",
+ "penaltyDuration": "30 sec",
+ "yieldDuration": "1 sec",
+ "bulletinLevel": "WARN",
+ "runDurationMillis": 0,
+ "concurrentlySchedulableTaskCount": 1,
+ "autoTerminatedRelationships": [
+ "original",
+ "failure"
+ ],
+ "scheduledState": "ENABLED",
+ "retryCount": 10000,
+ "retriedRelationships": [
+ "failure"
+ ],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "maxBackoffPeriod": "10 mins",
+ "componentType": "PROCESSOR",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "e427a2b2-355c-3ede-9749-b3e4a8561d22",
+ "instanceIdentifier": "e048d449-7da8-30f3-8ff4-71c87c1d73db",
+ "name": "ConsumeKafka",
+ "comments": "",
+ "position": {
+ "x": -250.0,
+ "y": -268.5
+ },
+ "type": "org.apache.nifi.kafka.processors.ConsumeKafka",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-kafka-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Topics": "#{Topic Names}",
+ "Commit Offsets": "true",
+ "Header Name Pattern": null,
+ "Key Format": "byte-array",
+ "Key Record Reader": null,
+ "Message Demarcator": null,
+ "Record Reader": "919b2a35-4bc7-380c-b592-6ad2ffffbed5",
+ "Key Attribute Encoding": "utf-8",
+ "Max Uncommitted Size": null,
+ "Topic Format": "names",
+ "Header Encoding": "UTF-8",
+ "Max Uncommitted Time": "5 secs",
+ "Kafka Connection Service": "ccf048dc-004a-35be-8e4d-c7a763a30b02",
+ "Separate By Key": "false",
+ "Processing Strategy": "RECORD",
+ "Record Writer": "2384a9ae-311e-3d5f-983b-13f82174244c",
+ "Group ID": "#{Consumer Group ID}",
+ "auto.offset.reset": "#{Kafka Auto Offset Reset}",
+ "Output Strategy": "USE_VALUE"
+ },
+ "propertyDescriptors": {
+ "Topics": {
+ "name": "Topics",
+ "displayName": "Topics",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Commit Offsets": {
+ "name": "Commit Offsets",
+ "displayName": "Commit Offsets",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Header Name Pattern": {
+ "name": "Header Name Pattern",
+ "displayName": "Header Name Pattern",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Key Format": {
+ "name": "Key Format",
+ "displayName": "Key Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Key Record Reader": {
+ "name": "Key Record Reader",
+ "displayName": "Key Record Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Message Demarcator": {
+ "name": "Message Demarcator",
+ "displayName": "Message Demarcator",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Record Reader": {
+ "name": "Record Reader",
+ "displayName": "Record Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Key Attribute Encoding": {
+ "name": "Key Attribute Encoding",
+ "displayName": "Key Attribute Encoding",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max Uncommitted Size": {
+ "name": "Max Uncommitted Size",
+ "displayName": "Max Uncommitted Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Topic Format": {
+ "name": "Topic Format",
+ "displayName": "Topic Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Header Encoding": {
+ "name": "Header Encoding",
+ "displayName": "Header Encoding",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max Uncommitted Time": {
+ "name": "Max Uncommitted Time",
+ "displayName": "Max Uncommitted Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Kafka Connection Service": {
+ "name": "Kafka Connection Service",
+ "displayName": "Kafka Connection Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Separate By Key": {
+ "name": "Separate By Key",
+ "displayName": "Separate By Key",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Processing Strategy": {
+ "name": "Processing Strategy",
+ "displayName": "Processing Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Record Writer": {
+ "name": "Record Writer",
+ "displayName": "Record Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Group ID": {
+ "name": "Group ID",
+ "displayName": "Group ID",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "auto.offset.reset": {
+ "name": "auto.offset.reset",
+ "displayName": "Auto Offset Reset",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Output Strategy": {
+ "name": "Output Strategy",
+ "displayName": "Output Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "style": {},
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "executionNode": "ALL",
+ "penaltyDuration": "30 sec",
+ "yieldDuration": "1 sec",
+ "bulletinLevel": "WARN",
+ "runDurationMillis": 0,
+ "concurrentlySchedulableTaskCount": 1,
+ "autoTerminatedRelationships": [
+ "parse failure"
+ ],
+ "scheduledState": "ENABLED",
+ "retryCount": 10,
+ "retriedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "maxBackoffPeriod": "10 mins",
+ "componentType": "PROCESSOR",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "501f25bb-cef8-328f-bfc2-fe58ad12e7b6",
+ "instanceIdentifier": "3de30158-1d3e-38d9-6e68-9578471aa26d",
+ "name": "PutS3Object",
+ "comments": "",
+ "position": {
+ "x": -250.0,
+ "y": 169.5
+ },
+ "type": "org.apache.nifi.processors.aws.s3.PutS3Object",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "FullControl User List": "${s3.permissions.full.users}",
+ "Owner": "${s3.owner}",
+ "proxy-configuration-service": null,
+ "Resource Transfer Source": "FLOWFILE_CONTENT",
+ "Cache Control": null,
+ "Endpoint Override URL": "#{S3 Endpoint Override URL}",
+ "File Resource Service": null,
+ "Multipart Upload Max Age Threshold": "7 days",
+ "Object Tags Prefix": null,
+ "Signer Override": "Default Signature",
+ "Content Type": "${mime.type}",
+ "Canned ACL": "${s3.permissions.cannedacl}",
+ "Use Path Style Access": "false",
+ "Encryption Service": null,
+ "Write ACL User List": "${s3.permissions.writeacl.users}",
+ "Use Chunked Encoding": "true",
+ "Read ACL User List": "${s3.permissions.readacl.users}",
+ "Content Disposition": null,
+ "Storage Class": "Standard",
+ "Multipart Part Size": "5 GB",
+ "Remove Tag Prefix": "false",
+ "Object Key": "${#{S3 Prefix}:append(\"/\"):replaceAll('^/$', '')}${filename}",
+ "AWS Credentials Provider service": "3f5da96c-9c54-39aa-8c84-e8098a44dfab",
+ "Multipart Threshold": "5 GB",
+ "Temporary Directory Multipart State": "${java.io.tmpdir}",
+ "Custom Signer Module Location": null,
+ "SSL Context Service": null,
+ "Server Side Encryption": "None",
+ "Bucket": "#{S3 Bucket}",
+ "Multipart Upload AgeOff Interval": "60 min",
+ "Write Permission User List": "${s3.permissions.write.users}",
+ "Communications Timeout": "30 secs",
+ "Region": "#{S3 Region}",
+ "Read Permission User List": "${s3.permissions.read.users}",
+ "Custom Signer Class Name": null,
+ "Expiration Time Rule": null
+ },
+ "propertyDescriptors": {
+ "FullControl User List": {
+ "name": "FullControl User List",
+ "displayName": "FullControl User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Owner": {
+ "name": "Owner",
+ "displayName": "Owner",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "proxy-configuration-service": {
+ "name": "proxy-configuration-service",
+ "displayName": "Proxy Configuration Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Resource Transfer Source": {
+ "name": "Resource Transfer Source",
+ "displayName": "Resource Transfer Source",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Cache Control": {
+ "name": "Cache Control",
+ "displayName": "Cache Control",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Endpoint Override URL": {
+ "name": "Endpoint Override URL",
+ "displayName": "Endpoint Override URL",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "File Resource Service": {
+ "name": "File Resource Service",
+ "displayName": "File Resource Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Upload Max Age Threshold": {
+ "name": "Multipart Upload Max Age Threshold",
+ "displayName": "Multipart Upload Max Age Threshold",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Object Tags Prefix": {
+ "name": "Object Tags Prefix",
+ "displayName": "Object Tags Prefix",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Signer Override": {
+ "name": "Signer Override",
+ "displayName": "Signer Override",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Content Type": {
+ "name": "Content Type",
+ "displayName": "Content Type",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Canned ACL": {
+ "name": "Canned ACL",
+ "displayName": "Canned ACL",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Use Path Style Access": {
+ "name": "Use Path Style Access",
+ "displayName": "Use Path Style Access",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Encryption Service": {
+ "name": "Encryption Service",
+ "displayName": "Encryption Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Write ACL User List": {
+ "name": "Write ACL User List",
+ "displayName": "Write ACL User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Use Chunked Encoding": {
+ "name": "Use Chunked Encoding",
+ "displayName": "Use Chunked Encoding",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Read ACL User List": {
+ "name": "Read ACL User List",
+ "displayName": "Read ACL User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Content Disposition": {
+ "name": "Content Disposition",
+ "displayName": "Content Disposition",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Storage Class": {
+ "name": "Storage Class",
+ "displayName": "Storage Class",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Part Size": {
+ "name": "Multipart Part Size",
+ "displayName": "Multipart Part Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Remove Tag Prefix": {
+ "name": "Remove Tag Prefix",
+ "displayName": "Remove Tag Prefix",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Object Key": {
+ "name": "Object Key",
+ "displayName": "Object Key",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "AWS Credentials Provider service": {
+ "name": "AWS Credentials Provider service",
+ "displayName": "AWS Credentials Provider service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Threshold": {
+ "name": "Multipart Threshold",
+ "displayName": "Multipart Threshold",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Temporary Directory Multipart State": {
+ "name": "Temporary Directory Multipart State",
+ "displayName": "Temporary Directory Multipart State",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Custom Signer Module Location": {
+ "name": "Custom Signer Module Location",
+ "displayName": "Custom Signer Module Location",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false,
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE",
+ "DIRECTORY"
+ ]
+ }
+ },
+ "SSL Context Service": {
+ "name": "SSL Context Service",
+ "displayName": "SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Server Side Encryption": {
+ "name": "Server Side Encryption",
+ "displayName": "Server Side Encryption",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Bucket": {
+ "name": "Bucket",
+ "displayName": "Bucket",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Upload AgeOff Interval": {
+ "name": "Multipart Upload AgeOff Interval",
+ "displayName": "Multipart Upload AgeOff Interval",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Write Permission User List": {
+ "name": "Write Permission User List",
+ "displayName": "Write Permission User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Communications Timeout": {
+ "name": "Communications Timeout",
+ "displayName": "Communications Timeout",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Region": {
+ "name": "Region",
+ "displayName": "Region",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Read Permission User List": {
+ "name": "Read Permission User List",
+ "displayName": "Read Permission User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Custom Signer Class Name": {
+ "name": "Custom Signer Class Name",
+ "displayName": "Custom Signer Class Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Expiration Time Rule": {
+ "name": "Expiration Time Rule",
+ "displayName": "Expiration Time Rule",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "style": {},
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "executionNode": "ALL",
+ "penaltyDuration": "30 sec",
+ "yieldDuration": "1 sec",
+ "bulletinLevel": "WARN",
+ "runDurationMillis": 0,
+ "concurrentlySchedulableTaskCount": 1,
+ "autoTerminatedRelationships": [
+ "success",
+ "failure"
+ ],
+ "scheduledState": "ENABLED",
+ "retryCount": 100000,
+ "retriedRelationships": [
+ "failure"
+ ],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "maxBackoffPeriod": "10 mins",
+ "componentType": "PROCESSOR",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ }
+ ],
+ "inputPorts": [],
+ "outputPorts": [],
+ "connections": [
+ {
+ "identifier": "4bd001fc-7fa6-3c34-b297-bae4987889ad",
+ "instanceIdentifier": "d6a7b460-571b-39b3-b55f-7d82f62b397a",
+ "name": "",
+ "source": {
+ "id": "e427a2b2-355c-3ede-9749-b3e4a8561d22",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "ConsumeKafka",
+ "comments": "",
+ "instanceIdentifier": "e048d449-7da8-30f3-8ff4-71c87c1d73db"
+ },
+ "destination": {
+ "id": "4ce9587d-a171-3dff-bea8-bc67f4b356cd",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "MergeRecord",
+ "comments": "",
+ "instanceIdentifier": "40e02539-8a55-3a44-53d9-e9d41f7db23a"
+ },
+ "labelIndex": 0,
+ "zIndex": 1,
+ "selectedRelationships": [
+ "success"
+ ],
+ "backPressureObjectThreshold": 10000,
+ "backPressureDataSizeThreshold": "1 GB",
+ "flowFileExpiration": "0 sec",
+ "prioritizers": [],
+ "bends": [],
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "partitioningAttribute": "",
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "componentType": "CONNECTION",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "08803783-a109-3362-9177-0f2016086a2a",
+ "instanceIdentifier": "45276df0-1076-39a8-7d84-097b7f593486",
+ "name": "",
+ "source": {
+ "id": "4ce9587d-a171-3dff-bea8-bc67f4b356cd",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "MergeRecord",
+ "comments": "",
+ "instanceIdentifier": "40e02539-8a55-3a44-53d9-e9d41f7db23a"
+ },
+ "destination": {
+ "id": "501f25bb-cef8-328f-bfc2-fe58ad12e7b6",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "PutS3Object",
+ "comments": "",
+ "instanceIdentifier": "3de30158-1d3e-38d9-6e68-9578471aa26d"
+ },
+ "labelIndex": 0,
+ "zIndex": 2,
+ "selectedRelationships": [
+ "merged"
+ ],
+ "backPressureObjectThreshold": 10000,
+ "backPressureDataSizeThreshold": "1 GB",
+ "flowFileExpiration": "0 sec",
+ "prioritizers": [],
+ "bends": [],
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "partitioningAttribute": "",
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "componentType": "CONNECTION",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ }
+ ],
+ "labels": [],
+ "funnels": [],
+ "controllerServices": [
+ {
+ "identifier": "cd697ee5-9621-3680-a6b8-af3237472b5b",
+ "instanceIdentifier": "2cdfec74-019a-1000-2786-edf80b919dfc",
+ "name": "Merge JSON Reader",
+ "comments": "",
+ "type": "org.apache.nifi.json.JsonTreeReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": null,
+ "Schema Text": "${avro.schema}",
+ "Max String Length": "20 MB",
+ "schema-application-strategy": "SELECTED_PART",
+ "Schema Registry": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "Schema Access Strategy": "schema-name",
+ "Schema Name": "${schema.name}",
+ "Timestamp Format": null,
+ "schema-inference-cache": null,
+ "Date Format": null,
+ "starting-field-strategy": "ROOT_NODE",
+ "Schema Version": null,
+ "starting-field-name": null,
+ "Time Format": null,
+ "Allow Comments": "false"
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max String Length": {
+ "name": "Max String Length",
+ "displayName": "Max String Length",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-application-strategy": {
+ "name": "schema-application-strategy",
+ "displayName": "Schema Application Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Timestamp Format": {
+ "name": "Timestamp Format",
+ "displayName": "Timestamp Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-inference-cache": {
+ "name": "schema-inference-cache",
+ "displayName": "Schema Inference Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Date Format": {
+ "name": "Date Format",
+ "displayName": "Date Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-strategy": {
+ "name": "starting-field-strategy",
+ "displayName": "Starting Field Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-name": {
+ "name": "starting-field-name",
+ "displayName": "Starting Field Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Time Format": {
+ "name": "Time Format",
+ "displayName": "Time Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Allow Comments": {
+ "name": "Allow Comments",
+ "displayName": "Allow Comments",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordReaderFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "b01590c6-5149-3c50-9d16-3a8e7b1bcdf6",
+ "instanceIdentifier": "2f0a61a3-b66d-3969-09d7-df35dd0b2740",
+ "name": "AvroRecordSetWriter",
+ "comments": "",
+ "type": "org.apache.nifi.avro.AvroRecordSetWriter",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": null,
+ "Schema Text": "${avro.schema}",
+ "Schema Registry": null,
+ "Schema Access Strategy": "inherit-record-schema",
+ "Schema Name": "${schema.name}",
+ "compression-format": "NONE",
+ "Schema Write Strategy": "avro-embedded",
+ "cache-size": "1000",
+ "Schema Cache": null,
+ "Schema Version": null,
+ "encoder-pool-size": "32",
+ "Schema Reference Writer": null
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "compression-format": {
+ "name": "compression-format",
+ "displayName": "Compression Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Write Strategy": {
+ "name": "Schema Write Strategy",
+ "displayName": "Schema Write Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "cache-size": {
+ "name": "cache-size",
+ "displayName": "Cache Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Cache": {
+ "name": "Schema Cache",
+ "displayName": "Schema Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "encoder-pool-size": {
+ "name": "encoder-pool-size",
+ "displayName": "Encoder Pool Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Writer": {
+ "name": "Schema Reference Writer",
+ "displayName": "Schema Reference Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordSetWriterFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "919b2a35-4bc7-380c-b592-6ad2ffffbed5",
+ "instanceIdentifier": "c169a376-4b21-3062-e8fb-1d243dd0e5df",
+ "name": "Kafka JSON Reader",
+ "comments": "",
+ "type": "org.apache.nifi.json.JsonTreeReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": "ecba6fe5-6beb-3b68-bad9-163a3f14c4c8",
+ "Schema Text": "${avro.schema}",
+ "Max String Length": "20 MB",
+ "schema-application-strategy": "SELECTED_PART",
+ "Schema Registry": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "Schema Access Strategy": "schema-reference-reader",
+ "Schema Name": "${schema.name}",
+ "Timestamp Format": null,
+ "schema-inference-cache": null,
+ "Date Format": null,
+ "starting-field-strategy": "ROOT_NODE",
+ "Schema Version": null,
+ "starting-field-name": null,
+ "Time Format": null,
+ "Allow Comments": "false"
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max String Length": {
+ "name": "Max String Length",
+ "displayName": "Max String Length",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-application-strategy": {
+ "name": "schema-application-strategy",
+ "displayName": "Schema Application Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Timestamp Format": {
+ "name": "Timestamp Format",
+ "displayName": "Timestamp Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-inference-cache": {
+ "name": "schema-inference-cache",
+ "displayName": "Schema Inference Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Date Format": {
+ "name": "Date Format",
+ "displayName": "Date Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-strategy": {
+ "name": "starting-field-strategy",
+ "displayName": "Starting Field Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-name": {
+ "name": "starting-field-name",
+ "displayName": "Starting Field Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Time Format": {
+ "name": "Time Format",
+ "displayName": "Time Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Allow Comments": {
+ "name": "Allow Comments",
+ "displayName": "Allow Comments",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordReaderFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "174d0040-161d-39dc-8da9-0f127fc46373",
+ "instanceIdentifier": "bf48cf99-1cae-375b-bfe1-bd9aa7368984",
+ "name": "AvroReader",
+ "comments": "",
+ "type": "org.apache.nifi.avro.AvroReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": "ecba6fe5-6beb-3b68-bad9-163a3f14c4c8",
+ "Schema Text": "${avro.schema}",
+ "cache-size": "1000",
+ "Schema Version": null,
+ "Schema Registry": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "Schema Access Strategy": "schema-reference-reader",
+ "Schema Name": "${schema.name}"
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "cache-size": {
+ "name": "cache-size",
+ "displayName": "Cache Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordReaderFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "3f5da96c-9c54-39aa-8c84-e8098a44dfab",
+ "instanceIdentifier": "7ea5a25b-89be-3652-9778-e30a69f84806",
+ "name": "AWSCredentialsProviderControllerService",
+ "comments": "",
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Assume Role STS Region": "us-west-2",
+ "Assume Role Proxy Configuration Service": null,
+ "Access Key ID": "#{S3 Access Key ID}",
+ "Assume Role ARN": null,
+ "Assume Role Session Time": "3600",
+ "Assume Role Session Name": null,
+ "Custom Signer Module Location": null,
+ "Use Default Credentials": "false",
+ "Assume Role External ID": null,
+ "Assume Role STS Endpoint Override": null,
+ "Profile Name": null,
+ "Credentials File": null,
+ "Custom Signer Class Name": null,
+ "Assume Role STS Signer Override": "Default Signature",
+ "Use Anonymous Credentials": "false",
+ "Secret Access Key": "#{S3 Secret Access Key}",
+ "Assume Role SSL Context Service": null
+ },
+ "propertyDescriptors": {
+ "Assume Role STS Region": {
+ "name": "Assume Role STS Region",
+ "displayName": "Assume Role STS Region",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role Proxy Configuration Service": {
+ "name": "Assume Role Proxy Configuration Service",
+ "displayName": "Assume Role Proxy Configuration Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Access Key ID": {
+ "name": "Access Key ID",
+ "displayName": "Access Key ID",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ },
+ "Assume Role ARN": {
+ "name": "Assume Role ARN",
+ "displayName": "Assume Role ARN",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role Session Time": {
+ "name": "Assume Role Session Time",
+ "displayName": "Assume Role Session Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role Session Name": {
+ "name": "Assume Role Session Name",
+ "displayName": "Assume Role Session Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Custom Signer Module Location": {
+ "name": "Custom Signer Module Location",
+ "displayName": "Custom Signer Module Location",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false,
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE",
+ "DIRECTORY"
+ ]
+ }
+ },
+ "Use Default Credentials": {
+ "name": "Use Default Credentials",
+ "displayName": "Use Default Credentials",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role External ID": {
+ "name": "Assume Role External ID",
+ "displayName": "Assume Role External ID",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role STS Endpoint Override": {
+ "name": "Assume Role STS Endpoint Override",
+ "displayName": "Assume Role STS Endpoint Override",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Profile Name": {
+ "name": "Profile Name",
+ "displayName": "Profile Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Credentials File": {
+ "name": "Credentials File",
+ "displayName": "Credentials File",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false,
+ "resourceDefinition": {
+ "cardinality": "SINGLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ }
+ },
+ "Custom Signer Class Name": {
+ "name": "Custom Signer Class Name",
+ "displayName": "Custom Signer Class Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role STS Signer Override": {
+ "name": "Assume Role STS Signer Override",
+ "displayName": "Assume Role STS Signer Override",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Use Anonymous Credentials": {
+ "name": "Use Anonymous Credentials",
+ "displayName": "Use Anonymous Credentials",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Secret Access Key": {
+ "name": "Secret Access Key",
+ "displayName": "Secret Access Key",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ },
+ "Assume Role SSL Context Service": {
+ "name": "Assume Role SSL Context Service",
+ "displayName": "Assume Role SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-service-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ },
+ {
+ "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-service-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "2384a9ae-311e-3d5f-983b-13f82174244c",
+ "instanceIdentifier": "d02eb508-0c95-31df-b86b-e1adadd77845",
+ "name": "JsonRecordSetWriter",
+ "comments": "",
+ "type": "org.apache.nifi.json.JsonRecordSetWriter",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": null,
+ "Schema Text": "${avro.schema}",
+ "Allow Scientific Notation": "false",
+ "compression-level": "1",
+ "Schema Registry": null,
+ "Schema Access Strategy": "inherit-record-schema",
+ "Schema Name": "${schema.name}",
+ "Timestamp Format": null,
+ "Date Format": null,
+ "Pretty Print JSON": "false",
+ "compression-format": "none",
+ "Schema Write Strategy": "schema-name",
+ "suppress-nulls": "never-suppress",
+ "output-grouping": "output-oneline",
+ "Schema Cache": null,
+ "Schema Version": null,
+ "Time Format": null,
+ "Schema Reference Writer": null
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Allow Scientific Notation": {
+ "name": "Allow Scientific Notation",
+ "displayName": "Allow Scientific Notation",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "compression-level": {
+ "name": "compression-level",
+ "displayName": "Compression Level",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Timestamp Format": {
+ "name": "Timestamp Format",
+ "displayName": "Timestamp Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Date Format": {
+ "name": "Date Format",
+ "displayName": "Date Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Pretty Print JSON": {
+ "name": "Pretty Print JSON",
+ "displayName": "Pretty Print JSON",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "compression-format": {
+ "name": "compression-format",
+ "displayName": "Compression Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Write Strategy": {
+ "name": "Schema Write Strategy",
+ "displayName": "Schema Write Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "suppress-nulls": {
+ "name": "suppress-nulls",
+ "displayName": "Suppress Null Values",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "output-grouping": {
+ "name": "output-grouping",
+ "displayName": "Output Grouping",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Cache": {
+ "name": "Schema Cache",
+ "displayName": "Schema Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Time Format": {
+ "name": "Time Format",
+ "displayName": "Time Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Writer": {
+ "name": "Schema Reference Writer",
+ "displayName": "Schema Reference Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordSetWriterFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "instanceIdentifier": "e0f7c153-e070-369b-451d-745b53ebe26a",
+ "name": "ConfluentSchemaRegistry",
+ "comments": "",
+ "type": "org.apache.nifi.confluent.schemaregistry.ConfluentSchemaRegistry",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-confluent-platform-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "SSL Context Service": null,
+ "Username": "#{Schema Registry Username}",
+ "Schema Registry URLs": "#{Schema Registry URLs}",
+ "Communications Timeout": "30 secs",
+ "Cache Size": "1000",
+ "Authentication Type": "#{Schema Registry Authentication Type}",
+ "Cache Expiration": "1 hour",
+ "Password": "#{Schema Registry Password}"
+ },
+ "propertyDescriptors": {
+ "SSL Context Service": {
+ "name": "SSL Context Service",
+ "displayName": "SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Username": {
+ "name": "Username",
+ "displayName": "Username",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry URLs": {
+ "name": "Schema Registry URLs",
+ "displayName": "Schema Registry URLs",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Communications Timeout": {
+ "name": "Communications Timeout",
+ "displayName": "Communications Timeout",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Cache Size": {
+ "name": "Cache Size",
+ "displayName": "Cache Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Authentication Type": {
+ "name": "Authentication Type",
+ "displayName": "Authentication Type",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Cache Expiration": {
+ "name": "Cache Expiration",
+ "displayName": "Cache Expiration",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Password": {
+ "name": "Password",
+ "displayName": "Password",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.schemaregistry.services.SchemaRegistry",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "ecba6fe5-6beb-3b68-bad9-163a3f14c4c8",
+ "instanceIdentifier": "da19419b-4821-315c-74b2-03d97cf411dc",
+ "name": "ConfluentEncodedSchemaReferenceReader",
+ "comments": "",
+ "type": "org.apache.nifi.confluent.schemaregistry.ConfluentEncodedSchemaReferenceReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-confluent-platform-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {},
+ "propertyDescriptors": {},
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.schemaregistry.services.SchemaReferenceReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "ccf048dc-004a-35be-8e4d-c7a763a30b02",
+ "instanceIdentifier": "5549d8f6-48eb-3677-821e-292b9b7e73d2",
+ "name": "Kafka3ConnectionService",
+ "comments": "",
+ "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-kafka-3-service-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "ack.wait.time": "10 sec",
+ "sasl.username": "#{Kafka SASL Username}",
+ "max.poll.records": "20000",
+ "bootstrap.servers": "#{Kafka Bootstrap Servers}",
+ "sasl.kerberos.service.name": null,
+ "security.protocol": "#{Kafka Security Protocol}",
+ "SSL Context Service": null,
+ "sasl.mechanism": "#{Kafka SASL Mechanism}",
+ "isolation.level": "read_committed",
+ "oauth2-access-token-provider-service": null,
+ "max.block.ms": "10 sec",
+ "kerberos-user-service": null,
+ "sasl.password": "#{Kafka SASL Password}",
+ "default.api.timeout.ms": "60 sec"
+ },
+ "propertyDescriptors": {
+ "ack.wait.time": {
+ "name": "ack.wait.time",
+ "displayName": "Acknowledgment Wait Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.username": {
+ "name": "sasl.username",
+ "displayName": "SASL Username",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max.poll.records": {
+ "name": "max.poll.records",
+ "displayName": "Max Poll Records",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "bootstrap.servers": {
+ "name": "bootstrap.servers",
+ "displayName": "Bootstrap Servers",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.kerberos.service.name": {
+ "name": "sasl.kerberos.service.name",
+ "displayName": "Kerberos Service Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "security.protocol": {
+ "name": "security.protocol",
+ "displayName": "Security Protocol",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "SSL Context Service": {
+ "name": "SSL Context Service",
+ "displayName": "SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.mechanism": {
+ "name": "sasl.mechanism",
+ "displayName": "SASL Mechanism",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "isolation.level": {
+ "name": "isolation.level",
+ "displayName": "Transaction Isolation Level",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "oauth2-access-token-provider-service": {
+ "name": "oauth2-access-token-provider-service",
+ "displayName": "OAuth2 Access Token Provider Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max.block.ms": {
+ "name": "max.block.ms",
+ "displayName": "Max Metadata Wait Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "kerberos-user-service": {
+ "name": "kerberos-user-service",
+ "displayName": "Kerberos User Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.password": {
+ "name": "sasl.password",
+ "displayName": "SASL Password",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ },
+ "default.api.timeout.ms": {
+ "name": "default.api.timeout.ms",
+ "displayName": "Client Timeout",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-kafka-service-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ }
+ ],
+ "parameterContextName": "Kafka to S3",
+ "defaultFlowFileExpiration": "0 sec",
+ "defaultBackPressureObjectThreshold": 10000,
+ "defaultBackPressureDataSizeThreshold": "1 GB",
+ "scheduledState": "ENABLED",
+ "executionEngine": "INHERITED",
+ "maxConcurrentTasks": 1,
+ "statelessFlowTimeout": "1 min",
+ "componentType": "PROCESS_GROUP",
+ "flowFileConcurrency": "UNBOUNDED",
+ "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE"
+ },
+ "externalControllerServices": {},
+ "parameterContexts": {
+ "Kafka to S3": {
+ "name": "Kafka to S3",
+ "parameters": [
+ {
+ "name": "S3 Secret Access Key",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Schema Registry Password",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Consumer Group ID",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Kafka SASL Mechanism",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "SCRAM-SHA-512"
+ },
+ {
+ "name": "Kafka Data Format",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "JSON"
+ },
+ {
+ "name": "Schema Registry Username",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "S3 Data Format",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "JSON"
+ },
+ {
+ "name": "Schema Registry URLs",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "http://localhost:8081"
+ },
+ {
+ "name": "Merge Latency",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "10 mins"
+ },
+ {
+ "name": "S3 Endpoint Override URL",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Schema Registry Authentication Type",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "BASIC"
+ },
+ {
+ "name": "Kafka Security Protocol",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "SASL_SSL"
+ },
+ {
+ "name": "Maximum Object size",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "300 MB"
+ },
+ {
+ "name": "S3 Region",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "us-west-2"
+ },
+ {
+ "name": "Kafka SASL Password",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Kafka Bootstrap Servers",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Kafka Auto Offset Reset",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "latest"
+ },
+ {
+ "name": "S3 Bucket",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Kafka SASL Username",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "S3 Access Key ID",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Target Object Size",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "256 MB"
+ },
+ {
+ "name": "Topic Names",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "S3 Prefix",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ }
+ ],
+ "inheritedParameterContexts": [],
+ "description": "",
+ "componentType": "PARAMETER_CONTEXT"
+ }
+ },
+ "flowEncodingVersion": "1.0",
+ "parameterProviders": {},
+ "latest": false
+}
\ No newline at end of file
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml
new file mode 100644
index 0000000000..c36971572c
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml
@@ -0,0 +1,226 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-integration-tests</artifactId>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>https://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Test Containers for Kafka and S3 -->
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers-kafka</artifactId>
+ <version>2.0.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>localstack</artifactId>
+ <version>1.20.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Kafka client for producing test data -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>3.9.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Confluent Schema Registry and Avro dependencies for testing -->
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-avro-serializer</artifactId>
+ <version>7.8.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>7.8.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.11.3</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- AWS SDK for S3 verification in tests -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <version>2.36.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>url-connection-client</artifactId>
+ <version>2.36.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Necessary NAR dependencies -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-service-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-service-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-shared-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-services-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-confluent-platform-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+
+ <!-- NARs required by framework -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-jetty-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-provenance-repository-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-server-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <!-- Copy nars into libDir for test harness -->
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-nar</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/libDir</outputDirectory>
+ <includeTypes>nar</includeTypes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/src/test/java/org/apache/nifi/connectors/kafkas3/KafkaToS3IT.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/src/test/java/org/apache/nifi/connectors/kafkas3/KafkaToS3IT.java
new file mode 100644
index 0000000000..77b748c74e
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/src/test/java/org/apache/nifi/connectors/kafkas3/KafkaToS3IT.java
@@ -0,0 +1,742 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import io.confluent.kafka.serializers.KafkaAvroSerializer;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.config.SaslConfigs;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.mock.connector.StandardConnectorTestRunner;
+import org.apache.nifi.mock.connector.server.ConnectorConfigVerificationResult;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.kafka.ConfluentKafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class KafkaToS3IT {
+
+ private static ConnectorTestRunner runner;
+ private static Network network;
+ private static ConfluentKafkaContainer kafkaContainer;
+ private static GenericContainer<?> schemaRegistryContainer;
+ private static LocalStackContainer localStackContainer;
+ private static S3Client s3Client;
+
+ private static final String SCRAM_USERNAME = "testuser";
+ private static final String SCRAM_PASSWORD = "testpassword";
+
+ private static final String S3_REGION = "us-west-2";
+
+ // JAAS configuration for Kafka broker SASL/PLAIN authentication.
+ // The 'username' and 'password' fields are credentials the broker uses for inter-broker communication.
+ // The 'user_<username>="<password>"' entries define client users that can authenticate to this broker.
+ // In this setup:
+ // - Broker uses 'admin' / 'admin-secret' for inter-broker communication (though we use PLAINTEXT for that)
+ // - Clients can authenticate using 'testuser' / 'testpassword' on the SASL listener with PLAIN mechanism
+ private static final String JAAS_CONFIG_CONTENT = """
+ KafkaServer {
+ org.apache.kafka.common.security.plain.PlainLoginModule required
+ username="admin"
+ password="admin-secret"
+ user_%s="%s";
+ };
+ """.formatted(SCRAM_USERNAME, SCRAM_PASSWORD);
+
+
+ @BeforeAll
+ public static void setupTestContainers() {
+ network = Network.newNetwork();
+
+ kafkaContainer = new ConfluentKafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.8.0"))
+ .withNetwork(network)
+ .withNetworkAliases("kafka")
+ .withStartupTimeout(Duration.ofSeconds(10))
+ .withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SASL:SASL_PLAINTEXT")
+ .withEnv("KAFKA_LISTENERS", "CONTROLLER://0.0.0.0:9094,BROKER://0.0.0.0:9092,PLAINTEXT://0.0.0.0:19092,SASL://0.0.0.0:9093")
+ .withEnv("KAFKA_ADVERTISED_LISTENERS", "BROKER://kafka:9092,PLAINTEXT://kafka:19092,SASL://localhost:9093")
+ .withEnv("KAFKA_CONTROLLER_LISTENER_NAMES", "CONTROLLER")
+ .withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER")
+ .withEnv("KAFKA_SASL_ENABLED_MECHANISMS", "PLAIN")
+ .withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0")
+ .withEnv("KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS", "1000")
+ .withEnv("KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS", "60000")
+ .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
+ .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1")
+ .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1")
+ .withEnv("KAFKA_OPTS", "-Djava.security.auth.login.config=/tmp/kafka_jaas.conf")
+ .withCommand(
+ "sh", "-c",
+ "echo '" + JAAS_CONFIG_CONTENT + "' > /tmp/kafka_jaas.conf && " +
+ "/etc/confluent/docker/run"
+ );
+
+ kafkaContainer.setPortBindings(List.of("9093:9093"));
+ kafkaContainer.start();
+
+ schemaRegistryContainer = new GenericContainer<>(DockerImageName.parse("confluentinc/cp-schema-registry:7.8.0"))
+ .withNetwork(network)
+ .withExposedPorts(8081)
+ .withEnv("SCHEMA_REGISTRY_HOST_NAME", "schema-registry")
+ .withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "PLAINTEXT://kafka:19092")
+ .withEnv("SCHEMA_REGISTRY_LISTENERS", "http://0.0.0.0:8081")
+ .withStartupTimeout(Duration.ofSeconds(60))
+ .waitingFor(Wait.forHttp("/subjects").forStatusCode(200))
+ .dependsOn(kafkaContainer);
+
+ schemaRegistryContainer.start();
+
+ localStackContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.1.0"))
+ .withServices(LocalStackContainer.Service.S3)
+ .withStartupTimeout(Duration.ofSeconds(30));
+
+ localStackContainer.start();
+
+ s3Client = S3Client.builder()
+ .endpointOverride(localStackContainer.getEndpoint())
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(localStackContainer.getAccessKey(), localStackContainer.getSecretKey())))
+ .region(Region.of(S3_REGION))
+ .httpClient(UrlConnectionHttpClient.builder().build())
+ .forcePathStyle(true)
+ .build();
+ }
+
+ @BeforeEach
+ public void setupRunner() {
+ runner = new StandardConnectorTestRunner.Builder()
+ .connectorClassName("org.apache.nifi.connectors.kafkas3.KafkaToS3")
+ .narLibraryDirectory(new File("target/libDir"))
+ .build();
+ assertNotNull(runner);
+ }
+
+ @AfterAll
+ public static void cleanupTestContainers() {
+ if (s3Client != null) {
+ s3Client.close();
+ }
+
+ if (localStackContainer != null) {
+ localStackContainer.stop();
+ }
+
+ if (schemaRegistryContainer != null) {
+ schemaRegistryContainer.stop();
+ }
+
+ if (kafkaContainer != null) {
+ kafkaContainer.stop();
+ }
+
+ if (network != null) {
+ network.close();
+ }
+ }
+
+ @AfterEach
+ public void cleanupRunner() throws IOException {
+ if (runner != null) {
+ runner.close();
+ }
+ }
+
+
+ private void createS3Bucket(final String bucketName) {
+ s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+ }
+
+ private void createKafkaTopics(final String... topicNames) throws ExecutionException, InterruptedException {
+ final Properties adminProps = new Properties();
+ adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9093");
+ adminProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ adminProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ adminProps.put(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";",
+ SCRAM_USERNAME, SCRAM_PASSWORD
+ ));
+
+ try (final AdminClient adminClient = AdminClient.create(adminProps)) {
+ final List<NewTopic> topics = new ArrayList<>();
+ for (final String topicName : topicNames) {
+ topics.add(new NewTopic(topicName, 1, (short) 1));
+ }
+
+ adminClient.createTopics(topics).all().get();
+ }
+ }
+
+ private void produceRecordsToTopic(final String topicName, final String... records) throws ExecutionException, InterruptedException {
+ final Properties producerProps = new Properties();
+ producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9093");
+ producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ producerProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ producerProps.put(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";",
+ SCRAM_USERNAME, SCRAM_PASSWORD
+ ));
+
+ try (final KafkaProducer<String, String> producer = new KafkaProducer<>(producerProps)) {
+ for (final String record : records) {
+ final ProducerRecord<String, String> producerRecord = new ProducerRecord<>(topicName, record);
+ producer.send(producerRecord).get();
+ }
+
+ producer.flush();
+ }
+ }
+
+ private String getSchemaRegistryUrl() {
+ return String.format("http://%s:%d", schemaRegistryContainer.getHost(), schemaRegistryContainer.getMappedPort(8081));
+ }
+
+ private void produceAvroRecordsToTopic(final String topicName, final Schema schema, final GenericRecord... records) throws ExecutionException, InterruptedException {
+ final Properties producerProps = new Properties();
+ producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9093");
+ producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
+ producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ producerProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ producerProps.put(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";",
+ SCRAM_USERNAME, SCRAM_PASSWORD
+ ));
+ producerProps.put("schema.registry.url", getSchemaRegistryUrl());
+
+ try (final KafkaProducer<String, GenericRecord> producer = new KafkaProducer<>(producerProps)) {
+ for (final GenericRecord record : records) {
+ final ProducerRecord<String, GenericRecord> producerRecord = new ProducerRecord<>(topicName, record);
+ producer.send(producerRecord).get();
+ }
+
+ producer.flush();
+ }
+ }
+
+
+ @Test
+ public void testVerification() throws ExecutionException, InterruptedException, FlowUpdateException {
+ createKafkaTopics("topic-1", "topic-2", "topic-3", "topic-4", "topic-5", "Z-topic", "an-important-topic");
+
+ produceRecordsToTopic("topic-1",
+ """
+ {"id": 1, "name": "Alice", "age": 30}""",
+ """
+ {"id": 2, "name": "Bob", "age": 25}""",
+ """
+ {"id": 3, "name": "Charlie", "age": 35}"""
+ );
+
+ produceRecordsToTopic("an-important-topic",
+ "This is a plaintext message",
+ "Another important message",
+ "Final plaintext record"
+ );
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ runner.applyUpdate();
+
+ // Perform verification to ensure that valid server configuration passes
+ final ConnectorConfigVerificationResult connectionVerificationResults = runner.verifyConfiguration("Kafka Connection", List.of(kafkaServerConfig));
+ connectionVerificationResults.assertNoFailures();
+
+ // Apply the configuration that we've now validated
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig));
+
+ // Perform verification to ensure that valid topic configuration passes
+ final PropertyGroupConfiguration topic1Config = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "topic-1",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSuccessfulFlow",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+ final ConnectorConfigVerificationResult topic1VerificationResults = runner.verifyConfiguration("Kafka Topics", List.of(topic1Config));
+ topic1VerificationResults.assertNoFailures();
+
+ // Perform verification against a topic with invalid data for the selected data format
+ final PropertyGroupConfiguration importantTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "an-important-topic",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSuccessfulFlow",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+
+ final ConnectorConfigVerificationResult importantTopicVerificationResults = runner.verifyConfiguration("Kafka Topics", List.of(importantTopicConfig));
+ final List<ConfigVerificationResult> invalidImportantTopicResults = importantTopicVerificationResults.getFailedResults();
+ assertEquals(1, invalidImportantTopicResults.size());
+ final ConfigVerificationResult invalidResult = invalidImportantTopicResults.getFirst();
+ assertTrue(invalidResult.getExplanation().contains("parse"), "Unexpected validation reason: " + invalidResult.getExplanation());
+
+ runner.applyUpdate();
+ }
+
+ @Test
+ public void testJsonFlow() throws IOException, ExecutionException, InterruptedException, FlowUpdateException {
+ final String bucketName = "test-json-flow";
+ createS3Bucket(bucketName);
+ createKafkaTopics("story");
+
+ produceRecordsToTopic("story",
+ """
+ {"page": 1, "words": "Once upon a time, there was a NiFi developer." }""",
+ """
+ {"page": 2, "words": "The developer wanted to build a connector to move data from Kafka to S3." }""",
+ """
+ {"page": 3, "words": "After much effort, the connector was complete and worked flawlessly!" }""",
+ """
+ {"page": 4, "words": "The end." }"""
+ );
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration kafkaTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "story",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSuccessfulFlow",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+
+ final PropertyGroupConfiguration s3DestinationConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "Avro",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+ final PropertyGroupConfiguration s3CredentialsConfig = PropertyGroupConfiguration.fromStringValues("S3 Credentials", Map.of(
+ "S3 Authentication Strategy", "Access Key ID and Secret Key",
+ "S3 Access Key ID", localStackContainer.getAccessKey(),
+ "S3 Secret Access Key", localStackContainer.getSecretKey()
+ ));
+ final PropertyGroupConfiguration s3MergeConfig = PropertyGroupConfiguration.fromStringValues("Merge Configuration", Map.of(
+ "Target Object Size", "1 MB",
+ "Merge Latency", "1 sec"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig));
+ runner.configure("Kafka Topics", List.of(kafkaTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3DestinationConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ final List<ValidationResult> validationResults = runner.validate();
+ assertEquals(Collections.emptyList(), validationResults);
+
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(10));
+ runner.waitForIdle(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ verifyS3ObjectsCreated(bucketName);
+ }
+
+ private void verifyS3ObjectsCreated(final String bucketName) throws IOException {
+ final ListObjectsV2Response listResponse = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> objects = listResponse.contents();
+
+ assertFalse(objects.isEmpty(), "Expected at least one object in S3 bucket");
+
+ for (final S3Object s3Object : objects) {
+ final GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(s3Object.key()).build();
+ try (final ResponseInputStream<GetObjectResponse> objectContent = s3Client.getObject(getObjectRequest)) {
+ final long objectSize = objectContent.response().contentLength();
+ assertTrue(objectSize > 0, "Expected S3 object " + s3Object.key() + " to have content");
+ }
+ }
+ }
+
+ @Test
+ public void testSchemaRegistryVerification() throws ExecutionException, InterruptedException, FlowUpdateException {
+ createKafkaTopics("avro-topic");
+
+ final String schemaString = """
+ {
+ "type": "record",
+ "name": "TestRecord",
+ "namespace": "org.apache.nifi.test",
+ "fields": [
+ {"name": "id", "type": "int"},
+ {"name": "message", "type": "string"}
+ ]
+ }""";
+
+ final Schema schema = new Schema.Parser().parse(schemaString);
+
+ final GenericRecord record1 = new GenericData.Record(schema);
+ record1.put("id", 100);
+ record1.put("message", "Test message 1");
+
+ final GenericRecord record2 = new GenericData.Record(schema);
+ record2.put("id", 200);
+ record2.put("message", "Test message 2");
+
+ produceAvroRecordsToTopic("avro-topic", schema, record1, record2);
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration schemaRegistryConfig = PropertyGroupConfiguration.fromStringValues("Schema Registry Settings", Map.of(
+ "Schema Registry URL", getSchemaRegistryUrl()
+ ));
+
+ final ConnectorConfigVerificationResult connectionVerificationResults = runner.verifyConfiguration("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+ connectionVerificationResults.assertNoFailures();
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+
+ final PropertyGroupConfiguration avroTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "avro-topic",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSchemaRegistryVerification",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "Avro"
+ ));
+
+ final ConnectorConfigVerificationResult avroTopicVerificationResults = runner.verifyConfiguration("Kafka Topics", List.of(avroTopicConfig));
+ avroTopicVerificationResults.assertNoFailures();
+
+ runner.applyUpdate();
+ }
+
+ @Test
+ public void testWithSchemaRegistry() throws IOException, ExecutionException, InterruptedException, FlowUpdateException {
+ final String bucketName = "test-schema-registry";
+ createS3Bucket(bucketName);
+ createKafkaTopics("user-events");
+
+ final String schemaString = """
+ {
+ "type": "record",
+ "name": "UserEvent",
+ "namespace": "org.apache.nifi.test",
+ "fields": [
+ {"name": "userId", "type": "int"},
+ {"name": "userName", "type": "string"},
+ {"name": "eventType", "type": "string"},
+ {"name": "timestamp", "type": "long"}
+ ]
+ }""";
+
+ final Schema schema = new Schema.Parser().parse(schemaString);
+
+ final GenericRecord record1 = new GenericData.Record(schema);
+ record1.put("userId", 1001);
+ record1.put("userName", "alice");
+ record1.put("eventType", "login");
+ record1.put("timestamp", System.currentTimeMillis());
+
+ final GenericRecord record2 = new GenericData.Record(schema);
+ record2.put("userId", 1002);
+ record2.put("userName", "bob");
+ record2.put("eventType", "purchase");
+ record2.put("timestamp", System.currentTimeMillis());
+
+ final GenericRecord record3 = new GenericData.Record(schema);
+ record3.put("userId", 1003);
+ record3.put("userName", "charlie");
+ record3.put("eventType", "logout");
+ record3.put("timestamp", System.currentTimeMillis());
+
+ produceAvroRecordsToTopic("user-events", schema, record1, record2, record3);
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration schemaRegistryConfig = PropertyGroupConfiguration.fromStringValues("Schema Registry Settings", Map.of(
+ "Schema Registry URL", getSchemaRegistryUrl()
+ ));
+
+ final PropertyGroupConfiguration kafkaTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "user-events",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSchemaRegistry",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "Avro"
+ ));
+
+ final PropertyGroupConfiguration s3DestinationConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "Avro",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+ final PropertyGroupConfiguration s3CredentialsConfig = PropertyGroupConfiguration.fromStringValues("S3 Credentials", Map.of(
+ "S3 Authentication Strategy", "Access Key ID and Secret Key",
+ "S3 Access Key ID", localStackContainer.getAccessKey(),
+ "S3 Secret Access Key", localStackContainer.getSecretKey()
+ ));
+ final PropertyGroupConfiguration s3MergeConfig = PropertyGroupConfiguration.fromStringValues("Merge Configuration", Map.of(
+ "Target Object Size", "1 MB",
+ "Merge Latency", "1 sec"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+ runner.configure("Kafka Topics", List.of(kafkaTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3DestinationConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ final List<ValidationResult> validationResults = runner.validate();
+ assertEquals(Collections.emptyList(), validationResults);
+
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(10));
+ runner.waitForIdle(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ verifyS3ObjectsCreated(bucketName);
+ }
+
+ @Test
+ public void testSwitchFromJsonToAvroS3Format() throws IOException, ExecutionException, InterruptedException, FlowUpdateException {
+ final String bucketName = "test-switch-formats";
+ createS3Bucket(bucketName);
+
+ // Add data to json and avro topics
+ createKafkaTopics("json", "avro");
+
+ produceRecordsToTopic("json",
+ """
+ {"id": 1, "type": "json", "data": "First JSON record"}""",
+ """
+ {"id": 2, "type": "json", "data": "Second JSON record"}""",
+ """
+ {"id": 3, "type": "json", "data": "Third JSON record"}"""
+ );
+
+ final String schemaString = """
+ {
+ "type": "record",
+ "name": "AvroRecord",
+ "namespace": "org.apache.nifi.test",
+ "fields": [
+ {"name": "id", "type": "int"},
+ {"name": "type", "type": "string"},
+ {"name": "data", "type": "string"}
+ ]
+ }""";
+
+ final Schema schema = new Schema.Parser().parse(schemaString);
+
+ final GenericRecord avroRecord1 = new GenericData.Record(schema);
+ avroRecord1.put("id", 10);
+ avroRecord1.put("type", "avro");
+ avroRecord1.put("data", "First Avro record");
+
+ final GenericRecord avroRecord2 = new GenericData.Record(schema);
+ avroRecord2.put("id", 20);
+ avroRecord2.put("type", "avro");
+ avroRecord2.put("data", "Second Avro record");
+
+ final GenericRecord avroRecord3 = new GenericData.Record(schema);
+ avroRecord3.put("id", 30);
+ avroRecord3.put("type", "avro");
+ avroRecord3.put("data", "Third Avro record");
+
+ produceAvroRecordsToTopic("avro", schema, avroRecord1, avroRecord2, avroRecord3);
+
+ // Configure Connector to consume from JSON Kafka topic and write to S3 in JSON format, but with an invalid S3 endpoint.
+ // This will cause the data to remain queued, since PutS3Object will fail to write the data.
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration schemaRegistryConfig = PropertyGroupConfiguration.fromStringValues("Schema Registry Settings", Map.of(
+ "Schema Registry URL", getSchemaRegistryUrl()
+ ));
+
+ final PropertyGroupConfiguration jsonTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "json",
+ "Consumer Group ID", "nifi-kafka-to-s3-testReconfiguration",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+
+ final PropertyGroupConfiguration s3InvalidDestinationConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "JSON",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", "http://invalid-s3-endpoint:9999"
+ ));
+ final PropertyGroupConfiguration s3CredentialsConfig = PropertyGroupConfiguration.fromStringValues("S3 Credentials", Map.of(
+ "S3 Authentication Strategy", "Access Key ID and Secret Key",
+ "S3 Access Key ID", localStackContainer.getAccessKey(),
+ "S3 Secret Access Key", localStackContainer.getSecretKey()
+ ));
+ final PropertyGroupConfiguration s3MergeConfig = PropertyGroupConfiguration.fromStringValues("Merge Configuration", Map.of(
+ "Target Object Size", "1 MB",
+ "Merge Latency", "1 sec"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig));
+ runner.configure("Kafka Topics", List.of(jsonTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3InvalidDestinationConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ // Run the Connector with the invalid S3 endpoint to queue the JSON data. Wait for data to be queued up.
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ // Apply configuration to specify the correct S3 endpoint. Keep S3 Data Format as JSON for now.
+ // This will allow the Connector to write data to S3 and properly drain the data when we switch the S3 format from JSON to Avro.
+ final PropertyGroupConfiguration s3ValidDestinationJsonConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "JSON",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+
+ runner.configure("S3 Configuration", List.of(s3ValidDestinationJsonConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ // Make sure there is no data in S3 yet.
+ final ListObjectsV2Response initialListingResponse = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> initialS3Objects = initialListingResponse.contents();
+ assertEquals(List.of(), initialS3Objects);
+
+ // Now change the S3 Data Format from JSON to Avro. This should trigger draining of the queued JSON data.
+ final PropertyGroupConfiguration s3ValidDestinationAvroConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "Avro",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+
+ // Configure to consume from the Avro topic and change S3 format to Avro
+ final PropertyGroupConfiguration avroTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "avro",
+ "Consumer Group ID", "nifi-kafka-to-s3-testReconfiguration",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "Avro"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+ runner.configure("Kafka Topics", List.of(avroTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3ValidDestinationAvroConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ // After draining, there should be one JSON file in S3.
+ final ListObjectsV2Response jsonOnlyListing = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> jsonObjects = jsonOnlyListing.contents();
+ assertEquals(1, jsonObjects.size());
+
+ // Start connector, wait for more data to be ingested, and then wait for the Connector to be idle.
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(30));
+ runner.waitForIdle(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ // Verify that there are two objects in S3: one in JSON format and one in Avro format.
+ final ListObjectsV2Response listResponse = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> objects = listResponse.contents();
+
+ assertEquals(2, objects.size(), "Expected exactly 2 objects in S3 bucket: one for JSON data and one for Avro data");
+
+ for (final S3Object s3Object : objects) {
+ final GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(s3Object.key()).build();
+ try (final ResponseInputStream<GetObjectResponse> objectContent = s3Client.getObject(getObjectRequest)) {
+ final long objectSize = objectContent.response().contentLength();
+ assertTrue(objectSize > 0, "Expected S3 object " + s3Object.key() + " to have content");
+ }
+ }
+ }
+
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/pom.xml
new file mode 100644
index 0000000000..76e6e97097
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-connector</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml
new file mode 100644
index 0000000000..0fbed3c9ea
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connectors</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-kafka-to-s3-connector</module>
+ <module>nifi-kafka-to-s3-nar</module>
+ <module>nifi-kafka-to-s3-integration-tests</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ </dependencies>
+ </dependencyManagement>
+</project>
\ No newline at end of file
diff --git a/nifi-connectors/pom.xml b/nifi-connectors/pom.xml
new file mode 100644
index 0000000000..4186b8904d
--- /dev/null
+++ b/nifi-connectors/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connectors</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-kafka-to-s3-bundle</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
index 0fd4ebbc7a..b0a38e8aca 100644
--- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
+++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
@@ -17,7 +17,9 @@
package org.apache.nifi.processors.aws.s3;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
@@ -26,6 +28,7 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.aws.AbstractAwsSyncProcessor;
+import org.apache.nifi.processors.aws.region.RegionUtil;
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.services.s3.S3Client;
@@ -34,6 +37,7 @@ import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -315,4 +319,11 @@ public abstract class AbstractS3Processor extends AbstractAwsSyncProcessor<S3Cli
attributes.put(S3_ENCRYPTION_STRATEGY, encryptionService.getStrategyName());
}
}
+
+ @ConnectorMethod(name = "getAvailableRegions", description = "Returns the list of available AWS regions")
+ public List<String> getAvailableRegions() {
+ return RegionUtil.getAwsRegionAllowableValues().stream()
+ .map(AllowableValue::getValue)
+ .toList();
+ }
}
diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java
index b5e35d8b3e..2626c41c8a 100644
--- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java
+++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java
@@ -100,7 +100,7 @@ public final class RegionUtil {
*
* @return All available AWS regions as AllowableValues.
*/
- private static List<AllowableValue> getAwsRegionAllowableValues() {
+ public static List<AllowableValue> getAwsRegionAllowableValues() {
final List<AllowableValue> values = new ArrayList<>();
for (final Region region : Region.regions()) {
values.add(createAwsRegionAllowableValue(region));
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
index 455c5ee5dc..a75a3ae50c 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
@@ -26,9 +26,11 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.Validator;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.kafka.processors.common.KafkaUtils;
import org.apache.nifi.kafka.processors.consumer.OffsetTracker;
@@ -58,11 +60,14 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.VerifiableProcessor;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.RecordSetWriterFactory;
import org.apache.nifi.util.StringUtils;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
@@ -512,26 +517,110 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess
final List<ConfigVerificationResult> verificationResults = new ArrayList<>();
final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
- final PollingContext pollingContext = createPollingContext(context);
- final KafkaConsumerService consumerService = connectionService.getConsumerService(pollingContext);
+ final PollingContext pollingContext = createPollingContext(context, null, AutoOffsetReset.EARLIEST);
+ try (final KafkaConsumerService consumerService = connectionService.getConsumerService(pollingContext)) {
+ final ConfigVerificationResult partitionVerification = verifyPartitions(consumerService, pollingContext);
+ verificationResults.add(partitionVerification);
+
+ final ConfigVerificationResult parsingResult = verifyCanParse(context, consumerService, verificationLogger);
+ verificationResults.add(parsingResult);
+ } catch (final IOException e) {
+ verificationResults.add(new ConfigVerificationResult.Builder()
+ .verificationStepName("Communicate with Kafka Broker")
+ .outcome(Outcome.FAILED)
+ .explanation("There was an I/O failure when communicating with Kafka: " + e)
+ .build());
+ }
+
+ return verificationResults;
+ }
- final ConfigVerificationResult.Builder verificationPartitions = new ConfigVerificationResult.Builder()
- .verificationStepName("Verify Topic Partitions");
+ private ConfigVerificationResult verifyPartitions(final KafkaConsumerService consumerService, final PollingContext pollingContext) {
+ final ConfigVerificationResult.Builder partitionVerification = new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Topic Partitions");
try {
final List<PartitionState> partitionStates = consumerService.getPartitionStates();
- verificationPartitions
- .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
- .explanation(String.format("Partitions [%d] found for Topics %s", partitionStates.size(), pollingContext.getTopics()));
+ partitionVerification
+ .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .explanation(String.format("Found [%d] partitions for Topics %s", partitionStates.size(), pollingContext.getTopics()));
} catch (final Exception e) {
getLogger().error("Topics {} Partition verification failed", pollingContext.getTopics(), e);
- verificationPartitions
- .outcome(ConfigVerificationResult.Outcome.FAILED)
- .explanation(String.format("Topics %s Partition access failed: %s", pollingContext.getTopics(), e));
+ partitionVerification
+ .outcome(ConfigVerificationResult.Outcome.FAILED)
+ .explanation(String.format("Topics %s Partition access failed: %s", pollingContext.getTopics(), e));
}
- verificationResults.add(verificationPartitions.build());
- return verificationResults;
+ return partitionVerification.build();
+ }
+
+ private ConfigVerificationResult verifyCanParse(final ProcessContext context, final KafkaConsumerService consumerService, final ComponentLog verificationLogger) {
+ final Iterable<ByteRecord> records = consumerService.poll(Duration.ofSeconds(60));
+ final ProcessingStrategy processingStrategy = context.getProperty(PROCESSING_STRATEGY).asAllowableValue(ProcessingStrategy.class);
+ if (processingStrategy != ProcessingStrategy.RECORD) {
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.SKIPPED)
+ .explanation("Processing Strategy is set to " + processingStrategy.getValue() + " so skipping record parsing verification")
+ .build();
+ }
+
+ final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
+ int recordIndex = 0;
+ for (final ByteRecord byteRecord : records) {
+ recordIndex++;
+ final Map<String, String> recordAttributes = KafkaUtils.toAttributes(
+ byteRecord, keyEncoding, headerNamePattern, headerEncoding, commitOffsets);
+
+ try (final InputStream inputStream = new ByteArrayInputStream(byteRecord.getValue());
+ final RecordReader reader = readerFactory.createRecordReader(recordAttributes, inputStream, byteRecord.getValue().length, verificationLogger)) {
+
+ while (reader.nextRecord() != null) {
+ }
+ } catch (final Exception e) {
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.FAILED)
+ .explanation("Failed to parse Record number " + recordIndex + ": " + e)
+ .build();
+ }
+ }
+
+ if (recordIndex == 0) {
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.SKIPPED)
+ .explanation("No records were received to parse")
+ .build();
+ }
+
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.SUCCESSFUL)
+ .explanation("Successfully parsed " + recordIndex + " records")
+ .build();
+ }
+
+
+ @ConnectorMethod(
+ name = "sampleTopics",
+ description = "Returns a list of sample data from the topics that would be consumed by this processor."
+ )
+ public List<byte[]> sampleTopics(final ProcessContext context) throws IOException {
+ final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
+ final PollingContext pollingContext = createPollingContext(context, "nifi-validation-" + System.currentTimeMillis(), AutoOffsetReset.EARLIEST);
+ try (final KafkaConsumerService consumerService = connectionService.getConsumerService(pollingContext)) {
+ final Iterable<ByteRecord> records = consumerService.poll(Duration.ofSeconds(60));
+ final List<byte[]> samples = new ArrayList<>();
+ for (final ByteRecord record : records) {
+ samples.add(record.getValue());
+ if (samples.size() >= 10) {
+ break;
+ }
+ }
+
+ return samples;
+ }
}
private KafkaConsumerService getConsumerService(final ProcessContext context) {
@@ -620,6 +709,10 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess
final String groupId = context.getProperty(GROUP_ID).getValue();
final String offsetReset = context.getProperty(AUTO_OFFSET_RESET).getValue();
final AutoOffsetReset autoOffsetReset = AutoOffsetReset.valueOf(offsetReset.toUpperCase());
+ return createPollingContext(context, groupId, autoOffsetReset);
+ }
+
+ private PollingContext createPollingContext(final ProcessContext context, final String groupId, final AutoOffsetReset autoOffsetReset) {
final String topics = context.getProperty(TOPICS).evaluateAttributeExpressions().getValue();
final String topicFormat = context.getProperty(TOPIC_FORMAT).getValue();
@@ -636,4 +729,5 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess
return pollingContext;
}
+
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java
index 531e5fff07..14c50d45ba 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java
@@ -17,6 +17,7 @@
package org.apache.nifi.kafka.processors;
import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
import org.apache.nifi.kafka.service.api.KafkaConnectionService;
import org.apache.nifi.kafka.service.api.common.PartitionState;
import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
@@ -36,6 +37,7 @@ import static org.apache.nifi.kafka.processors.ConsumeKafka.CONNECTION_SERVICE;
import static org.apache.nifi.kafka.processors.ConsumeKafka.GROUP_ID;
import static org.apache.nifi.kafka.processors.ConsumeKafka.TOPICS;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@@ -96,10 +98,15 @@ class ConsumeKafkaTest {
runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID);
final List<ConfigVerificationResult> results = processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap());
- assertEquals(1, results.size());
+ final List<ConfigVerificationResult> successResults = results.stream()
+ .filter(result -> result.getOutcome() == ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .toList();
+ assertEquals(1, successResults.size());
- final ConfigVerificationResult firstResult = results.iterator().next();
- assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, firstResult.getOutcome());
+ final boolean anyFailures = results.stream().anyMatch(result -> result.getOutcome() == Outcome.FAILED);
+ assertFalse(anyFailures, "At least one verification result was a failure: " + results);
+
+ final ConfigVerificationResult firstResult = successResults.getFirst();
assertNotNull(firstResult.getExplanation());
}
@@ -113,9 +120,14 @@ class ConsumeKafkaTest {
runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID);
final List<ConfigVerificationResult> results = processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap());
- assertEquals(1, results.size());
+ assertEquals(2, results.size());
+
+ final List<ConfigVerificationResult> failedResults = results.stream()
+ .filter(result -> result.getOutcome() == ConfigVerificationResult.Outcome.FAILED)
+ .toList();
+ assertEquals(1, failedResults.size());
- final ConfigVerificationResult firstResult = results.iterator().next();
+ final ConfigVerificationResult firstResult = failedResults.getFirst();
assertEquals(ConfigVerificationResult.Outcome.FAILED, firstResult.getOutcome());
assertNotNull(firstResult.getExplanation());
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java
index 27b07b41e3..171ff02fdf 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java
@@ -34,7 +34,7 @@ public class PollingContext {
public PollingContext(final String groupId, final Collection<String> topics,
final AutoOffsetReset autoOffsetReset) {
- this.groupId = Objects.requireNonNull(groupId, "Group ID required");
+ this.groupId = groupId;
this.topics = Collections.unmodifiableCollection(Objects.requireNonNull(topics, "Topics required"));
this.topicPattern = null;
this.autoOffsetReset = Objects.requireNonNull(autoOffsetReset, "Auto Offset Reset required");
@@ -42,7 +42,7 @@ public class PollingContext {
public PollingContext(final String groupId, final Pattern topicPattern,
final AutoOffsetReset autoOffsetReset) {
- this.groupId = Objects.requireNonNull(groupId, "Group ID required");
+ this.groupId = groupId;
this.topics = Collections.emptyList();
this.topicPattern = Objects.requireNonNull(topicPattern, "Topic Patten required");
this.autoOffsetReset = Objects.requireNonNull(autoOffsetReset, "Auto Offset Reset required");
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
index b29a444932..d4dfa6fd34 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
@@ -34,6 +34,8 @@ import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.ConfigVerificationResult;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
+import org.apache.nifi.components.connector.components.MethodArgument;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
@@ -46,6 +48,7 @@ import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
import org.apache.nifi.kafka.service.api.consumer.PollingContext;
import org.apache.nifi.kafka.service.api.producer.KafkaProducerService;
import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration;
+import org.apache.nifi.kafka.service.consumer.Kafka3AssignmentService;
import org.apache.nifi.kafka.service.consumer.Kafka3ConsumerService;
import org.apache.nifi.kafka.service.consumer.Subscription;
import org.apache.nifi.kafka.service.producer.Kafka3ProducerService;
@@ -70,6 +73,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -245,18 +250,25 @@ public class Kafka3ConnectionService extends AbstractControllerService implement
public KafkaConsumerService getConsumerService(final PollingContext pollingContext) {
Objects.requireNonNull(pollingContext, "Polling Context required");
- final Subscription subscription = createSubscription(pollingContext);
+ final String groupId = pollingContext.getGroupId();
final Properties properties = new Properties();
properties.putAll(consumerProperties);
- properties.put(ConsumerConfig.GROUP_ID_CONFIG, subscription.getGroupId());
- properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, subscription.getAutoOffsetReset().getValue());
+ if (groupId != null) {
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
+ }
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, pollingContext.getAutoOffsetReset().getValue());
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
final ByteArrayDeserializer deserializer = new ByteArrayDeserializer();
final Consumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties, deserializer, deserializer);
- return new Kafka3ConsumerService(getLogger(), consumer, subscription);
+ if (groupId == null) {
+ return new Kafka3AssignmentService(consumer, pollingContext.getTopics());
+ } else {
+ final Subscription subscription = createSubscription(pollingContext);
+ return new Kafka3ConsumerService(getLogger(), consumer, subscription);
+ }
}
private Subscription createSubscription(final PollingContext pollingContext) {
@@ -301,6 +313,26 @@ public class Kafka3ConnectionService extends AbstractControllerService implement
return uri;
}
+ @ConnectorMethod(
+ name = "listTopicNames",
+ description = "Returns a list of topic names available in the Kafka cluster",
+ arguments = {
+ @MethodArgument(name = "context", type = ConfigurationContext.class, description = "The configuration context that specifies connectivity details")
+ })
+ public List<String> listTopicNames(final ConfigurationContext context) throws ExecutionException, InterruptedException {
+ final Properties clientProperties = getClientProperties(context);
+ final Properties consumerProperties = getConsumerProperties(context, clientProperties);
+
+ try (final Admin admin = Admin.create(consumerProperties)) {
+ final ListTopicsResult result = admin.listTopics();
+ final Set<String> topicNames = result.names().get();
+ final List<String> sortedTopicNames = new ArrayList<>(topicNames);
+ sortedTopicNames.sort(String.CASE_INSENSITIVE_ORDER);
+ return sortedTopicNames;
+ }
+ }
+
+
@Override
public List<ConfigVerificationResult> verify(final ConfigurationContext configurationContext, final ComponentLog verificationLogger, final Map<String, String> variables) {
final List<ConfigVerificationResult> results = new ArrayList<>();
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3AssignmentService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3AssignmentService.java
new file mode 100644
index 0000000000..783134ca42
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3AssignmentService.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.kafka.service.consumer;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
+import org.apache.nifi.kafka.service.api.consumer.PollingSummary;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+public class Kafka3AssignmentService implements KafkaConsumerService {
+ private final Consumer<byte[], byte[]> consumer;
+ private volatile boolean closed = false;
+
+ public Kafka3AssignmentService(final Consumer<byte[], byte[]> consumer, final Collection<String> topicNames) {
+ this.consumer = consumer;
+
+ final List<TopicPartition> topicPartitions = new ArrayList<>();
+ for (final String topicName : topicNames) {
+ final List<PartitionInfo> partitionInfos = consumer.partitionsFor(topicName);
+ partitionInfos.forEach(info -> topicPartitions.add(new TopicPartition(info.topic(), info.partition())));
+ }
+
+ consumer.assign(topicPartitions);
+ }
+
+ @Override
+ public void commit(final PollingSummary pollingSummary) {
+ throw new UnsupportedOperationException("Commit not supported for assignment-based consumer");
+ }
+
+ @Override
+ public void rollback() {
+ // no-op
+ }
+
+ @Override
+ public boolean isClosed() {
+ return false;
+ }
+
+ @Override
+ public Iterable<ByteRecord> poll(final Duration maxWaitDuration) {
+ final ConsumerRecords<byte[], byte[]> consumerRecords = consumer.poll(maxWaitDuration);
+ if (consumerRecords.isEmpty()) {
+ return List.of();
+ }
+
+ return new RecordIterable(consumerRecords);
+ }
+
+ @Override
+ public List<PartitionState> getPartitionStates() {
+ final Set<TopicPartition> topicPartitions = consumer.assignment();
+ return topicPartitions.stream()
+ .map(tp -> new PartitionState(tp.topic(), tp.partition()))
+ .toList();
+ }
+
+ @Override
+ public void close() {
+ if (closed) {
+ return;
+ }
+
+ closed = true;
+ consumer.close();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
index 59ea23818a..b1152fb249 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
@@ -18,7 +18,6 @@ package org.apache.nifi.kafka.service.consumer;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
@@ -27,13 +26,11 @@ import org.apache.nifi.kafka.service.api.common.PartitionState;
import org.apache.nifi.kafka.service.api.common.TopicPartitionSummary;
import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
import org.apache.nifi.kafka.service.api.consumer.PollingSummary;
-import org.apache.nifi.kafka.service.api.header.RecordHeader;
import org.apache.nifi.kafka.service.api.record.ByteRecord;
import org.apache.nifi.logging.ComponentLog;
import java.io.Closeable;
import java.time.Duration;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -179,58 +176,4 @@ public class Kafka3ConsumerService implements KafkaConsumerService, Closeable, C
return offsets;
}
-
-
- private static class RecordIterable implements Iterable<ByteRecord> {
- private final Iterator<ByteRecord> records;
-
- private RecordIterable(final Iterable<ConsumerRecord<byte[], byte[]>> consumerRecords) {
- this.records = new RecordIterator(consumerRecords);
- }
-
- @Override
- public Iterator<ByteRecord> iterator() {
- return records;
- }
- }
-
- private static class RecordIterator implements Iterator<ByteRecord> {
- private final Iterator<ConsumerRecord<byte[], byte[]>> consumerRecords;
-
- private RecordIterator(final Iterable<ConsumerRecord<byte[], byte[]>> records) {
- this.consumerRecords = records.iterator();
- }
-
- @Override
- public boolean hasNext() {
- return consumerRecords.hasNext();
- }
-
- @Override
- public ByteRecord next() {
- final ConsumerRecord<byte[], byte[]> consumerRecord = consumerRecords.next();
- final List<RecordHeader> recordHeaders = new ArrayList<>();
- consumerRecord.headers().forEach(header -> {
- final RecordHeader recordHeader = new RecordHeader(header.key(), header.value());
- recordHeaders.add(recordHeader);
- });
-
- // Support Kafka tombstones
- byte[] value = consumerRecord.value();
- if (value == null) {
- value = new byte[0];
- }
-
- return new ByteRecord(
- consumerRecord.topic(),
- consumerRecord.partition(),
- consumerRecord.offset(),
- consumerRecord.timestamp(),
- recordHeaders,
- consumerRecord.key(),
- value,
- 1
- );
- }
- }
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/RecordIterable.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/RecordIterable.java
new file mode 100644
index 0000000000..714caa06db
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/RecordIterable.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.kafka.service.consumer;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class RecordIterable implements Iterable<ByteRecord> {
+ private final Iterator<ByteRecord> records;
+
+ public RecordIterable(final Iterable<ConsumerRecord<byte[], byte[]>> consumerRecords) {
+ this.records = new RecordIterator(consumerRecords);
+ }
+
+ @Override
+ public Iterator<ByteRecord> iterator() {
+ return records;
+ }
+
+ private static class RecordIterator implements Iterator<ByteRecord> {
+ private final Iterator<ConsumerRecord<byte[], byte[]>> consumerRecords;
+
+ private RecordIterator(final Iterable<ConsumerRecord<byte[], byte[]>> records) {
+ this.consumerRecords = records.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return consumerRecords.hasNext();
+ }
+
+ @Override
+ public ByteRecord next() {
+ final ConsumerRecord<byte[], byte[]> consumerRecord = consumerRecords.next();
+ final List<RecordHeader> recordHeaders = new ArrayList<>();
+ consumerRecord.headers().forEach(header -> {
+ final RecordHeader recordHeader = new RecordHeader(header.key(), header.value());
+ recordHeaders.add(recordHeader);
+ });
+
+ // Support Kafka tombstones
+ byte[] value = consumerRecord.value();
+ if (value == null) {
+ value = new byte[0];
+ }
+
+ return new ByteRecord(
+ consumerRecord.topic(),
+ consumerRecord.partition(),
+ consumerRecord.offset(),
+ consumerRecord.timestamp(),
+ recordHeaders,
+ consumerRecord.key(),
+ value,
+ 1
+ );
+ }
+ }
+}
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java
index 17f4564dea..d22096a95d 100644
--- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java
+++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java
@@ -40,32 +40,32 @@ import java.util.Set;
@Tags({"test", "load", "duplicate"})
@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription("Intended for load testing, this processor will create the configured number of copies of each incoming FlowFile. The original FlowFile as well as all "
-+ "generated copies are sent to the 'success' relationship. In addition, each FlowFile gets an attribute 'copy.index' set to the copy number, where the original FlowFile gets "
-+ "a value of zero, and all copies receive incremented integer values.")
+ + "generated copies are sent to the 'success' relationship. In addition, each FlowFile gets an attribute 'copy.index' set to the copy number, where the original FlowFile gets "
+ + "a value of zero, and all copies receive incremented integer values.")
@WritesAttributes({
- @WritesAttribute(attribute = "copy.index", description = "A zero-based incrementing integer value based on which copy the FlowFile is.")
+ @WritesAttribute(attribute = "copy.index", description = "A zero-based incrementing integer value based on which copy the FlowFile is.")
})
public class DuplicateFlowFile extends AbstractProcessor {
public static final String COPY_INDEX_ATTRIBUTE = "copy.index";
static final PropertyDescriptor NUM_COPIES = new PropertyDescriptor.Builder()
- .name("Number of Copies")
- .description("Specifies how many copies of each incoming FlowFile will be made")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .defaultValue("100")
- .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
- .build();
+ .name("Number of Copies")
+ .description("Specifies how many copies of each incoming FlowFile will be made")
+ .required(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .defaultValue("100")
+ .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
+ .build();
private static final List<PropertyDescriptor> PROPERTY_DESCRIPTORS = List.of(
NUM_COPIES
);
static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("The original FlowFile and all copies will be sent to this relationship")
- .build();
+ .name("success")
+ .description("The original FlowFile and all copies will be sent to this relationship")
+ .build();
private static final Set<Relationship> RELATIONSHIPS = Set.of(
REL_SUCCESS
diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
index 737fbe3884..3233465e3c 100644
--- a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
+++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
@@ -17,6 +17,8 @@
package org.apache.nifi.controller.queue;
+import java.util.concurrent.Future;
+
/**
* Represents the status of a Drop FlowFile Request that has been issued to
* a {@link FlowFileQueue}. When a queue is requested to drop its FlowFiles,
@@ -75,4 +77,9 @@ public interface DropFlowFileStatus {
* @return the reason that the state is set to a Failure state, or <code>null</code> if the state is not {@link DropFlowFileState#FAILURE}.
*/
String getFailureReason();
+
+ /**
+ * @return a Future that can be used to determine when the drop operation has completed
+ */
+ Future<Void> getCompletionFuture();
}
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
index 94a6f563b8..1405a49667 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
@@ -48,7 +48,7 @@ public class TestVolatileProvenanceRepository {
@Test
public void testAddAndGet() throws IOException {
- repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));
+ repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties((String) null));
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
@@ -81,7 +81,7 @@ public class TestVolatileProvenanceRepository {
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
- repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));
+ repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties((String) null));
final String uuid = "00000000-0000-0000-0000-000000000000";
final Map<String, String> attributes = new HashMap<>();
@@ -120,7 +120,7 @@ public class TestVolatileProvenanceRepository {
@Test
public void testSearchForInverseValue() throws InterruptedException {
- repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));
+ repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties((String) null));
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java b/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java
index c7fcf0ed41..b8c998bfb8 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java
@@ -44,7 +44,8 @@ public enum ResourceType {
Operation("/operation"),
Tenant("/tenants"),
ParameterContext("/parameter-contexts"),
- ParameterProvider("/parameter-providers");
+ ParameterProvider("/parameter-providers"),
+ Connector("/connectors");
final String value;
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java
new file mode 100644
index 0000000000..780d496a14
--- /dev/null
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.cluster.coordination.http.replication;
+
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+import jakarta.ws.rs.core.Response.Status.Family;
+import jakarta.ws.rs.core.Response.StatusType;
+import org.apache.nifi.components.connector.ConnectorRequestReplicator;
+import org.apache.nifi.components.connector.ConnectorState;
+import org.apache.nifi.controller.flow.FlowManager;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class ClusteredConnectorRequestReplicator implements ConnectorRequestReplicator {
+ private static final String GET = "GET";
+
+ private final Supplier<RequestReplicator> requestReplicatorSupplier;
+
+ public ClusteredConnectorRequestReplicator(final Supplier<RequestReplicator> requestReplicatorSupplier) {
+ this.requestReplicatorSupplier = Objects.requireNonNull(requestReplicatorSupplier, "Request Replicator Supplier required");
+ }
+
+ @Override
+ public ConnectorState getState(final String connectorId) throws IOException {
+ final RequestReplicator requestReplicator = getRequestReplicator();
+ final AsyncClusterResponse asyncResponse = requestReplicator.replicate(GET, URI.create("/nifi-api/connectors/" + connectorId + "/status"), null, Map.of());
+ try {
+ final Response response = asyncResponse.awaitMergedResponse().getClientResponse();
+ verifyResponse(response.getStatusInfo(), connectorId);
+
+ // TODO: Need a ConnectorStatusEntity? Need correct URI.
+ return response.readEntity(ConnectorState.class);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while waiting for response for status of Connector with ID " + connectorId, e);
+ }
+ }
+
+ @Override
+ public void setFlowManager(final FlowManager flowManager) {
+ }
+
+ private RequestReplicator getRequestReplicator() {
+ final RequestReplicator requestReplicator = requestReplicatorSupplier.get();
+ return Objects.requireNonNull(requestReplicator, "Request Replicator required");
+ }
+
+ private void verifyResponse(final StatusType responseStatusType, final String connectorId) throws IOException {
+ final int statusCode = responseStatusType.getStatusCode();
+ final String reason = responseStatusType.getReasonPhrase();
+
+ if (responseStatusType.getStatusCode() == Status.NOT_FOUND.getStatusCode()) {
+ throw new IllegalArgumentException("Connector with ID + " + connectorId + " does not exist");
+ }
+
+ final Family responseFamily = responseStatusType.getFamily();
+ if (responseFamily == Family.SERVER_ERROR) {
+ throw new IOException("Server-side error requesting State for Connector with ID + " + connectorId + ". Status code: " + statusCode + ", reason: " + reason);
+ }
+ if (responseFamily == Family.CLIENT_ERROR) {
+ throw new IOException("Client-side error requesting State for Connector with ID " + connectorId + ". Status code: " + statusCode + ", reason: " + reason);
+ }
+ if (responseFamily != Family.SUCCESSFUL) {
+ throw new IOException("Unexpected response code " + statusCode + " returned when requesting State for Connector with ID " + connectorId + ": " + reason);
+ }
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/ConnectorRequestReplicatorConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/ConnectorRequestReplicatorConfiguration.java
new file mode 100644
index 0000000000..4024321bd1
--- /dev/null
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/ConnectorRequestReplicatorConfiguration.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.framework.configuration;
+
+import org.apache.nifi.cluster.coordination.http.replication.ClusteredConnectorRequestReplicator;
+import org.apache.nifi.cluster.coordination.http.replication.RequestReplicator;
+import org.apache.nifi.components.connector.ConnectorRequestReplicator;
+import org.apache.nifi.components.connector.StandaloneConnectorRequestReplicator;
+import org.apache.nifi.util.NiFiProperties;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ConnectorRequestReplicatorConfiguration {
+
+ @Bean
+ public ConnectorRequestReplicator connectorRequestReplicator(
+ final NiFiProperties nifiProperties,
+ final ObjectProvider<RequestReplicator> requestReplicatorProvider) {
+
+ if (nifiProperties.isClustered()) {
+ // We have to use an ObjectProvider here and obtain a Supplier because of a circular dependency.
+ // The request replicator will not be available when created. However, it will be
+ // available before attempting to use the ConnectorRequestReplicator.
+ return new ClusteredConnectorRequestReplicator(requestReplicatorProvider::getIfAvailable);
+ }
+
+ return new StandaloneConnectorRequestReplicator();
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java
index ad4ded6321..93f313d255 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java
@@ -48,8 +48,6 @@ public class FrameworkClusterConfiguration {
private EventReporter eventReporter;
- private FlowController flowController;
-
private ClusterCoordinator clusterCoordinator;
private WebClientService webClientService;
@@ -59,17 +57,12 @@ public class FrameworkClusterConfiguration {
this.properties = properties;
}
- @Autowired
- public void setFlowController(final FlowController flowController) {
- this.flowController = flowController;
- }
-
@Autowired
public void setEventReporter(final EventReporter eventReporter) {
this.eventReporter = eventReporter;
}
- @Autowired
+ @Autowired(required = false)
public void setClusterCoordinator(final ClusterCoordinator clusterCoordinator) {
this.clusterCoordinator = clusterCoordinator;
}
@@ -106,7 +99,7 @@ public class FrameworkClusterConfiguration {
}
@Bean
- public ClusterDecommissionTask decommissionTask() {
+ public ClusterDecommissionTask decommissionTask(final FlowController flowController) {
return new ClusterDecommissionTask(clusterCoordinator, flowController);
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
index 3d5bc1ea09..4a8167c5a2 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
@@ -268,7 +268,7 @@ public class TestThreadPoolRequestReplicator {
when(coordinator.getConnectionStatus(Mockito.any(NodeIdentifier.class))).thenReturn(new NodeConnectionStatus(nodeId, NodeConnectionState.CONNECTED));
final AtomicInteger requestCount = new AtomicInteger(0);
- final NiFiProperties props = NiFiProperties.createBasicNiFiProperties(null);
+ final NiFiProperties props = NiFiProperties.createBasicNiFiProperties((String) null);
final MockReplicationClient client = new MockReplicationClient();
final RequestCompletionCallback requestCompletionCallback = (uri, method, responses) -> {
@@ -329,7 +329,7 @@ public class TestThreadPoolRequestReplicator {
final ClusterCoordinator coordinator = createClusterCoordinator();
final AtomicInteger requestCount = new AtomicInteger(0);
- final NiFiProperties props = NiFiProperties.createBasicNiFiProperties(null);
+ final NiFiProperties props = NiFiProperties.createBasicNiFiProperties((String) null);
final MockReplicationClient client = new MockReplicationClient();
final RequestCompletionCallback requestCompletionCallback = (uri, method, responses) -> {
@@ -529,7 +529,7 @@ public class TestThreadPoolRequestReplicator {
private void withReplicator(final WithReplicator function, final Status status, final long delayMillis, final RuntimeException failure,
final String expectedRequestChain, final String expectedProxiedEntityGroups) {
final ClusterCoordinator coordinator = createClusterCoordinator();
- final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties(null);
+ final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties((String) null);
final MockReplicationClient client = new MockReplicationClient();
final RequestCompletionCallback requestCompletionCallback = (uri, method, responses) -> {
};
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
index 4d0860bd9a..6e95624832 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
@@ -42,12 +42,18 @@ import org.apache.nifi.components.ConfigurableComponent;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.InvocationFailedException;
+import org.apache.nifi.components.connector.components.ComponentState;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
+import org.apache.nifi.components.connector.components.MethodArgument;
import org.apache.nifi.components.validation.ValidationState;
import org.apache.nifi.components.validation.ValidationStatus;
import org.apache.nifi.components.validation.ValidationTrigger;
import org.apache.nifi.connectable.Connectable;
+import org.apache.nifi.connectable.ConnectableFlowFileActivity;
import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.connectable.FlowFileActivity;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.controller.exception.ProcessorInstantiationException;
import org.apache.nifi.controller.scheduling.LifecycleState;
@@ -178,6 +184,8 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
private volatile BackoffMechanism backoffMechanism;
private volatile String maxBackoffPeriod;
+ private final ConnectableFlowFileActivity flowFileActivity = new ConnectableFlowFileActivity();
+
public StandardProcessorNode(final LoggableComponent<Processor> processor, final String uuid,
final ValidationContextFactory validationContextFactory, final ProcessScheduler scheduler,
final ControllerServiceProvider controllerServiceProvider, final ReloadComponent reloadComponent,
@@ -977,7 +985,9 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
}
@Override
- public List<ConfigVerificationResult> verifyConfiguration(final ProcessContext context, final ComponentLog logger, final Map<String, String> attributes, final ExtensionManager extensionManager) {
+ public List<ConfigVerificationResult> verifyConfiguration(final ProcessContext context, final ComponentLog logger, final Map<String, String> attributes, final ExtensionManager extensionManager,
+ final ParameterLookup parameterLookup) {
+
final List<ConfigVerificationResult> results = new ArrayList<>();
try {
@@ -985,7 +995,7 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
final long startNanos = System.nanoTime();
// Call super's verifyConfig, which will perform component validation
- results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), getProcessGroup().getParameterContext()));
+ results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), parameterLookup));
final long validationComplete = System.nanoTime();
// If any invalid outcomes from validation, we do not want to perform additional verification, because we only run additional verification when the component is valid.
@@ -1628,6 +1638,9 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
// Completion Timestamp is set to MAX_VALUE because we don't want to timeout until the task has a chance to run.
final AtomicLong completionTimestampRef = new AtomicLong(Long.MAX_VALUE);
+ // Mark current time as latest activity time so that we don't show as idle when the processor was stopped.
+ flowFileActivity.reset();
+
// Create a task to invoke the @OnScheduled annotation of the processor
final Callable<Void> startupTask = () -> {
final ScheduledState currentScheduleState = scheduledState.get();
@@ -1922,6 +1935,86 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
}
}
+ @Override
+ public List<ConnectorMethod> getConnectorMethods() {
+ return getConnectorMethods(getProcessor().getClass());
+ }
+
+ public Object invokeConnectorMethod(final String methodName, final Map<String, Object> arguments, final ProcessContext processContext) throws InvocationFailedException {
+ final ConfigurableComponent component = getComponent();
+
+ try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(getExtensionManager(), component.getClass(), getIdentifier())) {
+ final Method implementationMethod = discoverConnectorMethod(component.getClass(), methodName);
+ final MethodArgument[] methodArguments = getConnectorMethodArguments(methodName, implementationMethod, component);
+ final List<Object> argumentValues = new ArrayList<>();
+ for (final MethodArgument methodArgument : methodArguments) {
+ final Object argumentValue = arguments.get(methodArgument.name());
+ if (ProcessContext.class.equals(methodArgument.type())) {
+ continue;
+ }
+
+ if (argumentValue == null && methodArgument.required()) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the required argument '"
+ + methodArgument.name() + "' was not provided");
+ }
+
+ if (argumentValue != null && !(methodArgument.type().isAssignableFrom(argumentValue.getClass()))) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the argument '"
+ + methodArgument.name() + "' is of type " + argumentValue.getClass().getName() + " defined by " + argumentValue.getClass().getClassLoader()
+ + " but the method expects type " + methodArgument.type().getName() + " defined by " + methodArgument.type().getClassLoader());
+ }
+
+ argumentValues.add(argumentValue);
+ }
+
+ // Inject ProcessContext if the method signature supports it
+ final Class<?>[] argumentTypes = implementationMethod.getParameterTypes();
+ if (argumentTypes.length > 0 && ProcessContext.class.isAssignableFrom(argumentTypes[0])) {
+ argumentValues.addFirst(processContext);
+ }
+ if (argumentTypes.length > 1 && ProcessContext.class.isAssignableFrom(argumentTypes[argumentTypes.length - 1])) {
+ argumentValues.add(processContext);
+ }
+
+ try {
+ implementationMethod.setAccessible(true);
+ return implementationMethod.invoke(component, argumentValues.toArray());
+ } catch (final Exception e) {
+ throw new InvocationFailedException(e);
+ }
+ }
+ }
+
+ private MethodArgument[] getConnectorMethodArguments(final String methodName, final Method implementationMethod, final ConfigurableComponent component) throws InvocationFailedException {
+ if (implementationMethod == null) {
+ throw new InvocationFailedException("No such connector method '" + methodName + "' exists for " + component.getClass().getName());
+ }
+
+ final ConnectorMethod connectorMethodDefinition = implementationMethod.getAnnotation(ConnectorMethod.class);
+ final ComponentState[] componentStates = connectorMethodDefinition.allowedStates();
+ final ComponentState currentState = getComponentState();
+ final boolean validState = Set.of(componentStates).contains(currentState);
+ if (!validState) {
+ throw new IllegalStateException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because Processor is in state " + currentState
+ + " but the Connector Method does not allow invocation in this state");
+ }
+
+ final MethodArgument[] methodArguments = connectorMethodDefinition.arguments();
+ return methodArguments;
+ }
+
+ private ComponentState getComponentState() {
+ final ScheduledState scheduledState = getScheduledState();
+
+ return switch (scheduledState) {
+ case DISABLED -> ComponentState.PROCESSOR_DISABLED;
+ case STOPPED -> ComponentState.STOPPED;
+ case RUNNING, RUN_ONCE -> ComponentState.RUNNING;
+ case STARTING -> ComponentState.STARTING;
+ case STOPPING -> ComponentState.STOPPING;
+ };
+ }
+
/**
* Marks the processor as fully stopped, and completes any futures that are to be completed as a result
*/
@@ -2167,4 +2260,9 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
serviceNode.updateReference(this, descriptor);
}
}
+
+ @Override
+ public FlowFileActivity getFlowFileActivity() {
+ return flowFileActivity;
+ }
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java
index 50308465d2..a08000428c 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java
@@ -637,7 +637,7 @@ public abstract class AbstractFlowManager implements FlowManager {
throw new IllegalStateException("Cannot create Parameter Context with name '" + name + "' because a Parameter Context already exists with that name");
}
- final ParameterReferenceManager referenceManager = new StandardParameterReferenceManager(this);
+ final ParameterReferenceManager referenceManager = new StandardParameterReferenceManager(this::getRootGroup);
final ParameterContext parameterContext = new StandardParameterContext.Builder()
.id(id)
.name(name)
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java
index 7c8e8963b8..6e6c58bd92 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java
@@ -19,13 +19,13 @@ package org.apache.nifi.controller.flowanalysis;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.registry.flow.mapping.ComponentIdLookup;
import org.apache.nifi.registry.flow.mapping.FlowMappingOptions;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.registry.flow.mapping.VersionedComponentStateLookup;
public class FlowAnalysisUtil {
public static final String ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE = "*****";
- public static NiFiRegistryFlowMapper createMapper(ExtensionManager extensionManager) {
+ public static VersionedComponentFlowMapper createMapper(ExtensionManager extensionManager) {
final FlowMappingOptions flowMappingOptions = new FlowMappingOptions.Builder()
.mapPropertyDescriptors(true)
.mapControllerServiceReferencesToVersionedId(true)
@@ -36,7 +36,7 @@ public class FlowAnalysisUtil {
.mapAssetReferences(true)
.build();
- final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(extensionManager, flowMappingOptions) {
+ final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(extensionManager, flowMappingOptions) {
@Override
public String getGroupId(String groupId) {
return groupId;
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 72ee40dcf2..9c1029b78c 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -594,7 +594,10 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
// Update the FlowFile Repository
try {
final Collection<StandardRepositoryRecord> repoRecords = checkpoint.records.values();
- context.getFlowFileRepository().updateRepository((Collection) repoRecords);
+ if (!repoRecords.isEmpty()) {
+ context.getFlowFileRepository().updateRepository((Collection) repoRecords);
+ context.getConnectable().getFlowFileActivity().updateLatestActivityTime();
+ }
} catch (final IOException ioe) {
// if we fail to commit the session, we need to roll back
// the checkpoints as well because none of the checkpoints
@@ -972,7 +975,7 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
// Therefore, we create an Iterable that can iterate over each of these events, modifying them as needed, and returning them
// in the appropriate order. This prevents an unnecessary step of creating an intermediate List and adding all of those values
// to the List.
- // This is done in a similar veign to how Java 8's streams work, iterating over the events and returning a processed version
+ // This is done in a similar vein to how Java 8's streams work, iterating over the events and returning a processed version
// one-at-a-time as opposed to iterating over the entire Collection and putting the results in another Collection. However,
// we don't want to change the Framework to require Java 8 at this time, because it's not yet as prevalent as we would desire
final Map<String, FlowFileRecord> flowFileRecordMap = new HashMap<>();
@@ -981,6 +984,9 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
flowFileRecordMap.put(flowFile.getAttribute(CoreAttributes.UUID.key()), flowFile);
}
+ // Update connectable to indicate how much data was received and sent.
+ updateTransferCounts(recordsToSubmit);
+
final long commitNanos = System.nanoTime();
final List<ProvenanceEventRecord> autoTermEvents = checkpoint.autoTerminatedEvents;
final Iterable<ProvenanceEventRecord> iterable = new Iterable<>() {
@@ -1027,6 +1033,23 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
provenanceRepo.registerEvents(iterable);
}
+ private void updateTransferCounts(final Collection<ProvenanceEventRecord> events) {
+ int createdCount = 0;
+ long createdBytes = 0L;
+ for (final ProvenanceEventRecord event : events) {
+ if (event.getEventType() == ProvenanceEventType.CREATE) {
+ createdCount++;
+ createdBytes += event.getFileSize();
+ }
+ }
+
+ context.getConnectable().getFlowFileActivity().updateTransferCounts(
+ provenanceReporter.getFlowFilesReceived() + createdCount,
+ // For bytes, consider received and fetched; we don't include fetched in count, as the FlowFile count didn't change.
+ provenanceReporter.getBytesReceived() + provenanceReporter.getBytesFetched() + createdBytes,
+ provenanceReporter.getFlowFilesSent(),
+ provenanceReporter.getBytesSent());
+ }
private void updateEventContentClaims(final ProvenanceEventBuilder builder, final FlowFile flowFile, final StandardRepositoryRecord repoRecord) {
final ContentClaim originalClaim = repoRecord.getOriginalClaim();
@@ -1235,6 +1258,10 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
resetWriteClaims();
resetReadClaim();
+ if (!recordValues.isEmpty()) {
+ context.getConnectable().getFlowFileActivity().updateLatestActivityTime();
+ }
+
if (recordsToHandle.isEmpty()) {
LOG.trace("{} was rolled back, but no events were performed by this ProcessSession", this);
acknowledgeRecords();
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java
index 9b87040540..3cb82157ca 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java
@@ -17,6 +17,8 @@
package org.apache.nifi.controller.service;
import org.apache.commons.lang3.ClassUtils;
+import org.apache.nifi.components.connector.components.ComponentState;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.ControllerServiceProxyWrapper;
import org.apache.nifi.nar.ExtensionManager;
@@ -29,7 +31,6 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -38,15 +39,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class StandardControllerServiceInvocationHandler implements ControllerServiceInvocationHandler {
private static final Logger logger = LoggerFactory.getLogger(StandardControllerServiceInvocationHandler.class);
private static final Method PROXY_WRAPPER_GET_WRAPPED_METHOD;
-
- private static final Set<Method> validDisabledMethods;
static {
- // methods that are okay to be called when the service is disabled.
- final Set<Method> validMethods = new HashSet<>();
- validMethods.addAll(Arrays.asList(ControllerService.class.getMethods()));
- validMethods.addAll(Arrays.asList(Object.class.getMethods()));
- validDisabledMethods = Collections.unmodifiableSet(validMethods);
-
try {
PROXY_WRAPPER_GET_WRAPPED_METHOD = ControllerServiceProxyWrapper.class.getMethod("getWrapped");
} catch (final NoSuchMethodException e) {
@@ -54,6 +47,7 @@ public class StandardControllerServiceInvocationHandler implements ControllerSer
}
}
+ private final Set<Method> validDisabledMethods;
private final ControllerService originalService;
private final AtomicReference<ControllerServiceNode> serviceNodeHolder = new AtomicReference<>(null);
private final ExtensionManager extensionManager;
@@ -73,6 +67,34 @@ public class StandardControllerServiceInvocationHandler implements ControllerSer
this.extensionManager = extensionManager;
this.originalService = originalService;
this.serviceNodeHolder.set(serviceNode);
+ validDisabledMethods = determineDisabledAllowedMethods(originalService);
+ }
+
+ private Set<Method> determineDisabledAllowedMethods(final ControllerService implementation) {
+ // methods that are okay to be called when the service is disabled.
+ final Set<Method> validMethods = new HashSet<>();
+ validMethods.addAll(Arrays.asList(ControllerService.class.getMethods()));
+ validMethods.addAll(Arrays.asList(Object.class.getMethods()));
+
+ // If there are any Connector Methods that are allowed to be called when the service is STOPPED or STOPPING, add those as well.
+ // This allows us to perform actions from Connectors, such as obtaining a list of available resources, during Connector configuration, etc.
+ // while the service is currently stopped.
+ for (final Method method : implementation.getClass().getDeclaredMethods()) {
+ final ConnectorMethod connectorMethod = method.getAnnotation(ConnectorMethod.class);
+ if (connectorMethod == null) {
+ continue;
+ }
+
+ final ComponentState[] allowedStates = connectorMethod.allowedStates();
+ for (final ComponentState allowedState : allowedStates) {
+ if (allowedState == ComponentState.STOPPED || allowedState == ComponentState.STOPPING) {
+ validMethods.add(method);
+ break;
+ }
+ }
+ }
+
+ return validMethods;
}
@Override
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
index 18202bc10d..890542e703 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
@@ -35,6 +35,10 @@ import org.apache.nifi.components.ConfigurableComponent;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.InvocationFailedException;
+import org.apache.nifi.components.connector.components.ComponentState;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
+import org.apache.nifi.components.connector.components.MethodArgument;
import org.apache.nifi.components.validation.ValidationState;
import org.apache.nifi.components.validation.ValidationStatus;
import org.apache.nifi.components.validation.ValidationTrigger;
@@ -504,7 +508,7 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
@Override
public List<ConfigVerificationResult> verifyConfiguration(final ConfigurationContext context, final ComponentLog logger, final Map<String, String> variables,
- final ExtensionManager extensionManager) {
+ final ExtensionManager extensionManager, final ParameterLookup parameterLookup) {
final List<ConfigVerificationResult> results = new ArrayList<>();
@@ -513,7 +517,7 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
final long startNanos = System.nanoTime();
// Call super's verifyConfig, which will perform component validation
- results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), getProcessGroup() == null ? null : getProcessGroup().getParameterContext()));
+ results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), parameterLookup));
final long validationComplete = System.nanoTime();
// If any invalid outcomes from validation, we do not want to perform additional verification, because we only run additional verification when the component is valid.
@@ -675,6 +679,7 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
final String message = "Enabling %s rejected: Validation Status [%s] Errors %s".formatted(serviceNode, validationStatus, errors);
future.completeExceptionally(new IllegalStateException(message));
}
+
// Enable command rescheduled or rejected
return;
}
@@ -890,6 +895,88 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
}
}
+ // TODO: Refactor, this is the same between ProcessorNode / ControllerServiceNode except for getComponentState
+ @Override
+ public Object invokeConnectorMethod(final String methodName, final Map<String, Object> arguments, final ConfigurationContext configurationContext) throws InvocationFailedException {
+ final ConfigurableComponent component = getComponent();
+
+ try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(getExtensionManager(), component.getClass(), getIdentifier())) {
+ final Method implementationMethod = discoverConnectorMethod(component.getClass(), methodName);
+ final MethodArgument[] methodArguments = getConnectorMethodArguments(methodName, implementationMethod, component);
+ final List<Object> argumentValues = new ArrayList<>();
+ for (final MethodArgument methodArgument : methodArguments) {
+ final Object argumentValue = arguments.get(methodArgument.name());
+ if (ConfigurationContext.class.equals(methodArgument.type())) {
+ continue;
+ }
+
+ if (argumentValue == null && methodArgument.required()) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the required argument '"
+ + methodArgument.name() + "' was not provided");
+ }
+
+ if (argumentValue != null && !(methodArgument.type().isAssignableFrom(argumentValue.getClass()))) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the argument '"
+ + methodArgument.name() + "' is of type " + argumentValue.getClass().getName() + " defined by " + argumentValue.getClass().getClassLoader()
+ + " but the method expects type " + methodArgument.type().getName() + " defined by " + methodArgument.type().getClassLoader());
+ }
+
+ argumentValues.add(argumentValue);
+ }
+
+ // Inject ConfigurationContext if the method signature supports it
+ // TODO: Can we move away from Maps and instead just use reflection to get the arguments directly?
+ final Class<?>[] argumentTypes = implementationMethod.getParameterTypes();
+ if (argumentTypes.length > 0 && ConfigurationContext.class.isAssignableFrom(argumentTypes[0])) {
+ argumentValues.addFirst(configurationContext);
+ }
+ if (argumentTypes.length > 1 && ConfigurationContext.class.isAssignableFrom(argumentTypes[argumentTypes.length - 1])) {
+ argumentValues.add(configurationContext);
+ }
+
+ try {
+ implementationMethod.setAccessible(true);
+ return implementationMethod.invoke(component, argumentValues.toArray());
+ } catch (final Exception e) {
+ throw new InvocationFailedException(e);
+ }
+ }
+ }
+
+ @Override
+ public List<ConnectorMethod> getConnectorMethods() {
+ return getConnectorMethods(getControllerServiceImplementation().getClass());
+ }
+
+ private MethodArgument[] getConnectorMethodArguments(final String methodName, final Method implementationMethod, final ConfigurableComponent component) throws InvocationFailedException {
+ if (implementationMethod == null) {
+ throw new InvocationFailedException("No such connector method '" + methodName + "' exists for " + component.getClass().getName());
+ }
+
+ final ConnectorMethod connectorMethodDefinition = implementationMethod.getAnnotation(ConnectorMethod.class);
+ final ComponentState[] componentStates = connectorMethodDefinition.allowedStates();
+ final ComponentState currentState = getComponentState();
+ final boolean validState = Set.of(componentStates).contains(currentState);
+ if (!validState) {
+ throw new IllegalStateException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because Processor is in state " + currentState
+ + " but the Connector Method does not allow invocation in this state");
+ }
+
+ final MethodArgument[] methodArguments = connectorMethodDefinition.arguments();
+ return methodArguments;
+ }
+
+ private ComponentState getComponentState() {
+ final ControllerServiceState scheduledState = getState();
+
+ return switch (scheduledState) {
+ case DISABLED -> ComponentState.STOPPED;
+ case DISABLING -> ComponentState.STOPPING;
+ case ENABLING -> ComponentState.STARTING;
+ case ENABLED -> ComponentState.RUNNING;
+ };
+ }
+
@Override
protected void performFlowAnalysisOnThis() {
getValidationContextFactory().getFlowAnalyzer().ifPresent(flowAnalyzer -> flowAnalyzer.analyzeControllerService(this));
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
index 714a5e0bd2..e1dc434685 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
@@ -298,7 +298,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi
}
@Override
- public Future<Void> enableControllerServicesAsync(final Collection<ControllerServiceNode> serviceNodes) {
+ public CompletableFuture<Void> enableControllerServicesAsync(final Collection<ControllerServiceNode> serviceNodes) {
final CompletableFuture<Void> future = new CompletableFuture<>();
processScheduler.submitFrameworkTask(() -> {
try {
@@ -370,7 +370,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi
}
@Override
- public Future<Void> enableControllerServiceAndDependencies(final ControllerServiceNode serviceNode) {
+ public CompletableFuture<Void> enableControllerServiceAndDependencies(final ControllerServiceNode serviceNode) {
if (serviceNode.isActive()) {
logger.debug("Enabling of Controller Service {} triggered but service already enabled", serviceNode);
return CompletableFuture.completedFuture(null);
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java
index d173468086..69e8fb1754 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java
@@ -30,7 +30,7 @@ import org.apache.nifi.flow.VersionedPropertyDescriptor;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.registry.flow.FlowSnapshotContainer;
import org.apache.nifi.registry.flow.RegisteredFlowSnapshot;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import java.util.Collections;
import java.util.HashMap;
@@ -45,13 +45,13 @@ public class StandardControllerServiceResolver implements ControllerServiceResol
private final Authorizer authorizer;
private final FlowManager flowManager;
- private final NiFiRegistryFlowMapper flowMapper;
+ private final VersionedComponentFlowMapper flowMapper;
private final ControllerServiceProvider controllerServiceProvider;
private final ControllerServiceApiLookup controllerServiceApiLookup;
public StandardControllerServiceResolver(final Authorizer authorizer,
final FlowManager flowManager,
- final NiFiRegistryFlowMapper flowMapper,
+ final VersionedComponentFlowMapper flowMapper,
final ControllerServiceProvider controllerServiceProvider,
final ControllerServiceApiLookup controllerServiceApiLookup) {
this.authorizer = authorizer;
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java
index 7ea59751e8..3779f5b344 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java
@@ -290,6 +290,7 @@ public class VolatileBulletinRepository implements BulletinRepository {
case FLOW_ANALYSIS_RULE -> FLOW_ANALYSIS_RULE_BULLETIN_STORE_KEY;
case PARAMETER_PROVIDER -> PARAMETER_PROVIDER_BULLETIN_STORE_KEY;
case FLOW_REGISTRY_CLIENT -> FLOW_REGISTRY_CLIENT_STORE_KEY;
+ case CONNECTOR -> bulletin.getSourceId();
default -> bulletin.getGroupId();
};
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
index b79ee4d6e8..4a8a6ab461 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
@@ -112,7 +112,7 @@ import org.apache.nifi.registry.flow.diff.FlowDifference;
import org.apache.nifi.registry.flow.diff.StandardComparableDataFlow;
import org.apache.nifi.registry.flow.diff.StandardFlowComparator;
import org.apache.nifi.registry.flow.diff.StaticDifferenceDescriptor;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.RemoteGroupPort;
import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor;
@@ -311,7 +311,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
@Override
public void synchronize(final ProcessGroup group, final VersionedExternalFlow versionedExternalFlow, final FlowSynchronizationOptions options) {
- final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(context.getExtensionManager(), context.getFlowMappingOptions());
+ final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(context.getExtensionManager(), context.getFlowMappingOptions());
final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(group, context.getControllerServiceProvider(), context.getFlowManager(), true);
final ComparableDataFlow localFlow = new StandardComparableDataFlow("Currently Loaded Flow", versionedGroup);
@@ -998,7 +998,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
private <T extends Connectable> Map<String, T> componentsById(final ProcessGroup group, final Function<ProcessGroup, Collection<T>> retrieveComponents) {
return retrieveComponents.apply(group).stream()
.collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier())), Function.identity()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier())), Function.identity()));
}
private <T> Map<String, T> componentsById(final ProcessGroup group, final Function<ProcessGroup, Collection<T>> retrieveComponents,
@@ -1006,7 +1006,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
return retrieveComponents.apply(group).stream()
.collect(Collectors.toMap(component -> retrieveVersionedComponentId.apply(component).orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(retrieveId.apply(component))), Function.identity()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(retrieveId.apply(component))), Function.identity()));
}
private void synchronizeFunnels(final ProcessGroup group, final VersionedProcessGroup proposed, final Map<String, Funnel> funnelsByVersionedId) {
@@ -1242,7 +1242,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final Map<String, Port> removedInputPortsByVersionId = new HashMap<>();
group.getInputPorts()
.forEach(port -> removedInputPortsByVersionId.put(port.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
+ VersionedComponentFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
flowContents.getInputPorts().stream()
.map(VersionedPort::getIdentifier)
@@ -1261,7 +1261,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final Map<String, Port> removedOutputPortsByVersionId = new HashMap<>();
group.getOutputPorts()
.forEach(port -> removedOutputPortsByVersionId.put(port.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
+ VersionedComponentFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
flowContents.getOutputPorts().stream()
.map(VersionedPort::getIdentifier)
@@ -1288,7 +1288,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
group.findAllProcessors()
.forEach(proc -> proposedProcessors.remove(proc.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(proc.getIdentifier()))));
+ VersionedComponentFlowMapper.generateVersionedComponentId(proc.getIdentifier()))));
for (final VersionedProcessor processorToAdd : proposedProcessors.values()) {
final String processorToAddClass = processorToAdd.getType();
@@ -1319,7 +1319,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
group.findAllControllerServices()
.forEach(service -> proposedServices.remove(service.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(service.getIdentifier()))));
+ VersionedComponentFlowMapper.generateVersionedComponentId(service.getIdentifier()))));
for (final VersionedControllerService serviceToAdd : proposedServices.values()) {
final String serviceToAddClass = serviceToAdd.getType();
@@ -1348,7 +1348,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
group.findAllConnections()
.forEach(conn -> proposedConnections.remove(conn.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(conn.getIdentifier()))));
+ VersionedComponentFlowMapper.generateVersionedComponentId(conn.getIdentifier()))));
for (final VersionedConnection connectionToAdd : proposedConnections.values()) {
if (connectionToAdd.getPrioritizers() != null) {
@@ -3030,7 +3030,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
private String getServiceInstanceId(final String serviceVersionedComponentId, final ProcessGroup group) {
for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) {
final String versionedId = serviceNode.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
if (versionedId.equals(serviceVersionedComponentId)) {
return serviceNode.getIdentifier();
}
@@ -3650,7 +3650,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final String rpgId = connectableComponent.getGroupId();
final Optional<RemoteProcessGroup> rpgOption = group.getRemoteProcessGroups().stream()
.filter(component -> rpgId.equals(component.getIdentifier()) || rpgId.equals(component.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
+ VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
.findAny();
if (rpgOption.isEmpty()) {
@@ -3676,7 +3676,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final String rpgId = connectableComponent.getGroupId();
final Optional<RemoteProcessGroup> rpgOption = group.getRemoteProcessGroups().stream()
.filter(component -> rpgId.equals(component.getIdentifier()) || rpgId.equals(component.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
+ VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
.findAny();
if (rpgOption.isEmpty()) {
@@ -3877,12 +3877,12 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
}
private <T extends org.apache.nifi.components.VersionedComponent & Connectable> boolean matchesId(final T component, final String id) {
- return id.equals(component.getIdentifier()) || id.equals(component.getVersionedComponentId().orElse(NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier())));
+ return id.equals(component.getIdentifier()) || id.equals(component.getVersionedComponentId().orElse(VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier())));
}
private boolean matchesGroupId(final ProcessGroup group, final String groupId) {
return groupId.equals(group.getIdentifier()) || group.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(group.getIdentifier())).equals(groupId);
+ VersionedComponentFlowMapper.generateVersionedComponentId(group.getIdentifier())).equals(groupId);
}
private void findAllProcessors(final Set<VersionedProcessor> processors, final Set<VersionedProcessGroup> childGroups, final Map<String, VersionedProcessor> map) {
@@ -3934,7 +3934,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
// match group's current connections to proposed connections to determine if they've been removed
for (final Connection connection : processGroup.getConnections()) {
final String versionedId = connection.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(connection.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(connection.getIdentifier()));
final VersionedConnection proposedConnection = proposedConnectionsByVersionedId.get(versionedId);
if (proposedConnection == null) {
// connection doesn't exist in proposed connections, make sure it doesn't have any data in it
@@ -3953,7 +3953,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
// match current child groups to proposed child groups to determine if they've been removed
for (final ProcessGroup childGroup : processGroup.getProcessGroups()) {
final String versionedId = childGroup.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(childGroup.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(childGroup.getIdentifier()));
final VersionedProcessGroup proposedChildGroup = proposedGroupsByVersionedId.get(versionedId);
if (proposedChildGroup == null) {
if (verifyConnectionRemoval) {
@@ -3979,7 +3979,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) {
final String serviceNodeVersionedComponentId = serviceNode.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
if (serviceNodeVersionedComponentId.equals(versionedComponentId)) {
return serviceNode;
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index bc8d717645..9fbfde6f0a 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -34,11 +34,13 @@ import org.apache.nifi.components.validation.ValidationStatus;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.connectable.FlowFileActivity;
import org.apache.nifi.connectable.Funnel;
import org.apache.nifi.connectable.LocalPort;
import org.apache.nifi.connectable.Port;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.connectable.Positionable;
+import org.apache.nifi.connectable.ProcessGroupFlowFileActivity;
import org.apache.nifi.controller.ComponentNode;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerService;
@@ -105,7 +107,7 @@ import org.apache.nifi.registry.flow.diff.StandardComparableDataFlow;
import org.apache.nifi.registry.flow.diff.StandardFlowComparator;
import org.apache.nifi.registry.flow.mapping.ComponentIdLookup;
import org.apache.nifi.registry.flow.mapping.FlowMappingOptions;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.registry.flow.mapping.VersionedComponentStateLookup;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.RemoteGroupPort;
@@ -138,6 +140,7 @@ import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -196,6 +199,8 @@ public final class StandardProcessGroup implements ProcessGroup {
private volatile ExecutionEngine executionEngine = ExecutionEngine.INHERITED;
private volatile int maxConcurrentTasks = 1;
private volatile String statelessFlowTimeout = "1 min";
+ private volatile Authorizable explicitParentAuthorizable;
+ private final FlowFileActivity flowFileActivity = new ProcessGroupFlowFileActivity(this);
private FlowFileConcurrency flowFileConcurrency = FlowFileConcurrency.UNBOUNDED;
private volatile FlowFileGate flowFileGate = new UnboundedFlowFileGate();
@@ -295,7 +300,7 @@ public final class StandardProcessGroup implements ProcessGroup {
@Override
public Authorizable getParentAuthorizable() {
- return getParent();
+ return explicitParentAuthorizable == null ? getParent() : explicitParentAuthorizable;
}
@Override
@@ -1720,7 +1725,7 @@ public final class StandardProcessGroup implements ProcessGroup {
}
@Override
- public Future<Void> startProcessor(final ProcessorNode processor, final boolean failIfStopping) {
+ public CompletableFuture<Void> startProcessor(final ProcessorNode processor, final boolean failIfStopping) {
readLock.lock();
try {
if (getProcessor(processor.getIdentifier()) == null) {
@@ -3931,7 +3936,7 @@ public final class StandardProcessGroup implements ProcessGroup {
// because this allows us to find the Controller Service when doing a Flow Diff.
ancestorServiceIds = parentGroup.getControllerServices(true).stream()
.map(cs -> cs.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(cs.getIdentifier())))
+ VersionedComponentFlowMapper.generateVersionedComponentId(cs.getIdentifier())))
.collect(Collectors.toSet());
}
@@ -3976,7 +3981,7 @@ public final class StandardProcessGroup implements ProcessGroup {
}
try {
- final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(extensionManager);
+ final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(extensionManager);
final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(this, controllerServiceProvider, flowManager, false);
final ComparableDataFlow currentFlow = new StandardComparableDataFlow("Local Flow", versionedGroup);
@@ -4551,6 +4556,48 @@ public final class StandardProcessGroup implements ProcessGroup {
return statelessFlowTimeout;
}
+ @Override
+ public FlowFileActivity getFlowFileActivity() {
+ return flowFileActivity;
+ }
+
+ @Override
+ public void setExplicitParentAuthorizable(final Authorizable parent) {
+ this.explicitParentAuthorizable = parent;
+ }
+
+ @Override
+ public CompletableFuture<Void> purge() {
+ final CompletableFuture<Void> purgeFuture = new CompletableFuture<>();
+
+ final Thread purgeThread = Thread.ofVirtual().name("Purge " + this).start(() -> {
+ try {
+ stopProcessing().get();
+ controllerServiceProvider.disableControllerServicesAsync(getControllerServices(true)).get();
+ purgeQueues();
+ removeComponents(this);
+
+ purgeFuture.complete(null);
+ } catch (final Throwable t) {
+ purgeFuture.completeExceptionally(t);
+ }
+ });
+
+ return purgeFuture;
+ }
+
+ private void purgeQueues() throws ExecutionException, InterruptedException {
+ for (final Connection connection : getConnections()) {
+ final FlowFileQueue flowFileQueue = connection.getFlowFileQueue();
+ if (flowFileQueue.isEmpty()) {
+ continue;
+ }
+
+ final DropFlowFileStatus status = connection.getFlowFileQueue().dropFlowFiles("purge-queues-" + getIdentifier(), "Framework");
+ status.getCompletionFuture().get();
+ }
+ }
+
@Override
public void setStatelessFlowTimeout(final String statelessFlowTimeout) {
if (statelessFlowTimeout == null) {
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ConnectorLogObserver.java
similarity index 72%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
copy to nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ConnectorLogObserver.java
index 7209492ca3..d6eba91f98 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ConnectorLogObserver.java
@@ -14,26 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.nifi.logging;
-import org.apache.nifi.controller.ProcessorNode;
+import org.apache.nifi.components.connector.ConnectorNode;
import org.apache.nifi.events.BulletinFactory;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.Severity;
-/**
- * Observes processor log messages and converts them into Bulletins.
- */
-public class ProcessorLogObserver implements LogObserver {
-
+public class ConnectorLogObserver implements LogObserver {
private static final String CATEGORY = "Log Message";
private final BulletinRepository bulletinRepository;
- private final ProcessorNode processorNode;
+ private final ConnectorNode connectorNode;
- public ProcessorLogObserver(BulletinRepository bulletinRepository, ProcessorNode processorNode) {
+ public ConnectorLogObserver(final BulletinRepository bulletinRepository, final ConnectorNode connectorNode) {
this.bulletinRepository = bulletinRepository;
- this.processorNode = processorNode;
+ this.connectorNode = connectorNode;
}
@Override
@@ -41,14 +38,12 @@ public class ProcessorLogObserver implements LogObserver {
// Map LogLevel.WARN to Severity.WARNING so that we are consistent with the Severity enumeration. Else, just use whatever
// the LogLevel is (INFO and ERROR map directly and all others we will just accept as they are).
final String bulletinLevel = (message.getLogLevel() == LogLevel.WARN) ? Severity.WARNING.name() : message.getLogLevel().toString();
- bulletinRepository.addBulletin(
- BulletinFactory.createBulletin(processorNode, CATEGORY, bulletinLevel, message.getMessage(), message.getFlowFileUuid(), message.getThrowable())
- );
+ bulletinRepository.addBulletin(BulletinFactory.createBulletin(connectorNode, CATEGORY, bulletinLevel, message.getMessage()));
}
@Override
public String getComponentDescription() {
- return processorNode.toString();
+ return connectorNode.toString();
}
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
index 7209492ca3..597b3c8c10 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
@@ -31,7 +31,7 @@ public class ProcessorLogObserver implements LogObserver {
private final BulletinRepository bulletinRepository;
private final ProcessorNode processorNode;
- public ProcessorLogObserver(BulletinRepository bulletinRepository, ProcessorNode processorNode) {
+ public ProcessorLogObserver(final BulletinRepository bulletinRepository, final ProcessorNode processorNode) {
this.bulletinRepository = bulletinRepository;
this.processorNode = processorNode;
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java
index a05d5c57ab..1944f12751 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java
@@ -22,7 +22,6 @@ import org.apache.nifi.controller.ComponentNode;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.PropertyConfiguration;
-import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.groups.ProcessGroup;
@@ -33,14 +32,15 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
+import java.util.function.Supplier;
public class StandardParameterReferenceManager implements ParameterReferenceManager {
- private final FlowManager flowManager;
+ private final Supplier<ProcessGroup> rootGroupLookup;
// TODO: Consider reworking this so that we don't have to recurse through all components all the time and instead
// have a 'caching' impl that AbstractComponentNode.setProperties() adds to/subtracts from.
- public StandardParameterReferenceManager(final FlowManager flowManager) {
- this.flowManager = flowManager;
+ public StandardParameterReferenceManager(final Supplier<ProcessGroup> rootGroupLookup) {
+ this.rootGroupLookup = rootGroupLookup;
}
@Override
@@ -57,14 +57,14 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana
public List<ParameterReferencedControllerServiceData> getReferencedControllerServiceData(final ParameterContext parameterContext, final String parameterName) {
final List<ParameterReferencedControllerServiceData> referencedControllerServiceData = new ArrayList<>();
+ final ProcessGroup rootGroup = rootGroupLookup.get();
final String versionedServiceId = parameterContext.getParameter(parameterName)
.map(Parameter::getValue)
- .map(this.flowManager::getControllerServiceNode)
+ .map(serviceId -> rootGroup.findControllerService(serviceId, true, true))
.flatMap(VersionedComponent::getVersionedComponentId)
.orElse(null);
- final ProcessGroup rootGroup = flowManager.getRootGroup();
final List<ProcessGroup> referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext));
for (final ProcessGroup group : referencingGroups) {
@@ -90,7 +90,7 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana
@Override
public Set<ProcessGroup> getProcessGroupsBound(final ParameterContext parameterContext) {
- final ProcessGroup rootGroup = flowManager.getRootGroup();
+ final ProcessGroup rootGroup = rootGroupLookup.get();
final List<ProcessGroup> referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext));
return new HashSet<>(referencingGroups);
@@ -100,14 +100,13 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana
final Function<ProcessGroup, Collection<T>> componentFunction) {
final Set<T> referencingComponents = new HashSet<>();
- final ProcessGroup rootGroup = flowManager.getRootGroup();
+ final ProcessGroup rootGroup = rootGroupLookup.get();
final List<ProcessGroup> referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext));
for (final ProcessGroup group : referencingGroups) {
for (final T componentNode : componentFunction.apply(group)) {
if (componentNode.isReferencingParameter(parameterName)) {
referencingComponents.add(componentNode);
- continue;
}
}
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java
index 5edcb7c026..f82b8eb24b 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java
@@ -37,7 +37,7 @@ import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.expression.ExpressionLanguageCompiler;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.parameter.Parameter;
-import org.apache.nifi.parameter.ParameterContext;
+import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.parameter.ParameterReference;
import java.util.Collection;
@@ -59,7 +59,7 @@ public class StandardValidationContext implements ValidationContext {
private final String annotationData;
private final String groupId;
private final String componentId;
- private final ParameterContext parameterContext;
+ private final ParameterLookup parameterLookup;
private final AtomicReference<Map<PropertyDescriptor, String>> effectiveValuesRef = new AtomicReference<>();
private final boolean validateConnections;
@@ -69,7 +69,7 @@ public class StandardValidationContext implements ValidationContext {
final String annotationData,
final String groupId,
final String componentId,
- final ParameterContext parameterContext,
+ final ParameterLookup parameterLookup,
final boolean validateConnections) {
this.controllerServiceProvider = controllerServiceProvider;
@@ -77,7 +77,7 @@ public class StandardValidationContext implements ValidationContext {
this.annotationData = annotationData;
this.groupId = groupId;
this.componentId = componentId;
- this.parameterContext = parameterContext;
+ this.parameterLookup = parameterLookup;
this.validateConnections = validateConnections;
preparedQueries = new HashMap<>(properties.size());
@@ -85,7 +85,7 @@ public class StandardValidationContext implements ValidationContext {
final PropertyDescriptor desc = entry.getKey();
final PropertyConfiguration configuration = entry.getValue();
- String value = (configuration == null) ? null : configuration.getEffectiveValue(parameterContext);
+ String value = (configuration == null) ? null : configuration.getEffectiveValue(parameterLookup);
if (value == null) {
value = desc.getDefaultValue();
}
@@ -103,12 +103,12 @@ public class StandardValidationContext implements ValidationContext {
@Override
public PropertyValue newPropertyValue(final String rawValue) {
final ResourceContext resourceContext = new StandardResourceContext(new StandardResourceReferenceFactory(), null);
- return new StandardPropertyValue(resourceContext, rawValue, controllerServiceProvider, parameterContext, Query.prepareWithParametersPreEvaluated(rawValue));
+ return new StandardPropertyValue(resourceContext, rawValue, controllerServiceProvider, parameterLookup, Query.prepareWithParametersPreEvaluated(rawValue));
}
@Override
public ExpressionLanguageCompiler newExpressionLanguageCompiler() {
- return new StandardExpressionLanguageCompiler(parameterContext);
+ return new StandardExpressionLanguageCompiler(parameterLookup);
}
@Override
@@ -123,9 +123,9 @@ public class StandardValidationContext implements ValidationContext {
@Override
public PropertyValue getProperty(final PropertyDescriptor property) {
final PropertyConfiguration configuredValue = properties.get(property);
- final String effectiveValue = configuredValue == null ? property.getDefaultValue() : configuredValue.getEffectiveValue(parameterContext);
+ final String effectiveValue = configuredValue == null ? property.getDefaultValue() : configuredValue.getEffectiveValue(parameterLookup);
final ResourceContext resourceContext = new StandardResourceContext(new StandardResourceReferenceFactory(), property);
- return new StandardPropertyValue(resourceContext, effectiveValue, controllerServiceProvider, parameterContext, preparedQueries.get(property));
+ return new StandardPropertyValue(resourceContext, effectiveValue, controllerServiceProvider, parameterLookup, preparedQueries.get(property));
}
@Override
@@ -139,7 +139,7 @@ public class StandardValidationContext implements ValidationContext {
for (final Map.Entry<PropertyDescriptor, PropertyConfiguration> entry : this.properties.entrySet()) {
final PropertyDescriptor descriptor = entry.getKey();
final PropertyConfiguration configuration = entry.getValue();
- final String value = configuration == null ? descriptor.getDefaultValue() : configuration.getEffectiveValue(parameterContext);
+ final String value = configuration == null ? descriptor.getDefaultValue() : configuration.getEffectiveValue(parameterLookup);
valueMap.put(entry.getKey(), value);
}
@@ -215,20 +215,20 @@ public class StandardValidationContext implements ValidationContext {
@Override
public boolean isParameterDefined(final String parameterName) {
- if (parameterContext == null) {
+ if (parameterLookup == null) {
return false;
}
- return parameterContext.getParameter(parameterName).isPresent();
+ return parameterLookup.getParameter(parameterName).isPresent();
}
@Override
public boolean isParameterSet(final String parameterName) {
- if (parameterContext == null) {
+ if (parameterLookup == null) {
return false;
}
- final Optional<Parameter> parameterOption = parameterContext.getParameter(parameterName);
+ final Optional<Parameter> parameterOption = parameterLookup.getParameter(parameterName);
if (!parameterOption.isPresent()) {
return false;
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java
index 09f23d12b9..9f0a1ef8cb 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java
@@ -22,7 +22,7 @@ import org.apache.nifi.controller.PropertyConfiguration;
... 14982 lines suppressed ...