You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2026/01/22 14:08:25 UTC
(nifi) 01/40: NIFI-15258: Initial implementation of Connectors
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch NIFI-15258
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 9d87f0947b4558413fa2602fe75ef06edb5482fd
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Tue Sep 2 18:22:14 2025 -0400
NIFI-15258: Initial implementation of Connectors
---
.cursorrules | 71 +
.../c2/command/DefaultFlowStateStrategyTest.java | 8 +-
nifi-commons/nifi-connector-utils/pom.xml | 42 +
.../connector/util/VersionedFlowUtils.java | 366 ++++
.../java/org/apache/nifi/util/NiFiProperties.java | 54 +-
nifi-commons/pom.xml | 1 +
.../nifi-connector-mock-api/pom.xml | 35 +
.../server/ConnectorConfigVerificationResult.java | 21 +-
.../mock/connector/server/ConnectorMockServer.java | 23 +-
.../mock/connector/server/ConnectorTestRunner.java | 52 +
.../nifi-connector-mock-server-nar/pom.xml | 55 +
.../nifi-connector-mock-server/pom.xml | 73 +
.../mock/connector/server/MockAuditService.java | 63 +
.../server/MockConnectorInitializationContext.java | 156 ++
.../connector/server/MockConnectorRepository.java | 25 +-
.../server/MockExtensionDiscoveringManager.java | 60 +
.../mock/connector/server/MockExtensionMapper.java | 44 +
.../server/MockRuleViolationsManager.java | 79 +
.../server/MockServerConfigVerificationResult.java | 52 +
.../mock/connector/server/MockStateManager.java | 89 +
.../connector/server/MockStateManagerProvider.java | 59 +
.../nifi/mock/connector/server/MockStateMap.java | 56 +
.../connector/server/NopPropertyEncryptor.java | 26 +-
.../mock/connector/server/PermitAllAuthorizer.java | 46 +
.../server/StandardConnectorMockServer.java | 304 +++
.../META-INF/services/org.apache.nifi.NiFiServer | 5 +-
...e.nifi.components.connector.ConnectorRepository | 5 +-
.../server/MockStateManagerProviderTest.java | 267 +++
.../nifi-connector-mock-integration-tests/pom.xml | 118 ++
.../mock/connectors/tests/CreateConnectorIT.java | 30 +-
.../mock/connectors/tests/MockProcessorIT.java | 91 +
.../pom.xml | 35 +
.../nifi-connector-mock-test-connectors/pom.xml | 48 +
.../nifi/mock/connectors/GenerateAndLog.java | 56 +
.../org.apache.nifi.components.connector.Connector | 5 +-
.../main/resources/flows/Generate_and_Update.json | 1 +
.../nifi-connector-mock-test-bundle/pom.xml | 33 +
.../nifi-connector-mock/pom.xml | 147 ++
.../connector/StandardConnectorTestRunner.java | 193 ++
.../src/main/resources/nifi.properties | 159 ++
nifi-connector-mock-bundle/pom.xml | 34 +
.../nifi-kafka-to-s3-connector/pom.xml | 40 +
.../connectors/kafkas3/KafkaConnectionStep.java | 135 ++
.../apache/nifi/connectors/kafkas3/KafkaToS3.java | 295 +++
.../connectors/kafkas3/KafkaToS3FlowBuilder.java | 217 ++
.../nifi/connectors/kafkas3/KafkaTopicsStep.java | 85 +
.../org/apache/nifi/connectors/kafkas3/S3Step.java | 156 ++
.../org.apache.nifi.components.connector.Connector | 5 +-
.../src/main/resources/flows/Kafka_to_S3.json | 2105 ++++++++++++++++++++
.../nifi-kafka-to-s3-integration-tests/pom.xml | 226 +++
.../nifi/connectors/kafkas3/KafkaToS3IT.java | 742 +++++++
.../nifi-kafka-to-s3-nar/pom.xml | 36 +
nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml | 39 +
nifi-connectors/pom.xml | 42 +
.../processors/aws/s3/AbstractS3Processor.java | 12 +
.../nifi/processors/aws/region/RegionUtil.java | 2 +-
.../apache/nifi/kafka/processors/ConsumeKafka.java | 118 +-
.../nifi/kafka/processors/ConsumeKafkaTest.java | 23 +-
.../kafka/service/api/consumer/PollingContext.java | 4 +-
.../kafka/service/Kafka3ConnectionService.java | 40 +-
.../service/consumer/Kafka3AssignmentService.java | 93 +
.../service/consumer/Kafka3ConsumerService.java | 57 -
.../kafka/service/consumer/RecordIterable.java | 79 +
.../processors/standard/DuplicateFlowFile.java | 2 +-
.../nifi/controller/queue/DropFlowFileStatus.java | 7 +
.../TestVolatileProvenanceRepository.java | 6 +-
.../nifi/authorization/resource/ResourceType.java | 3 +-
.../ClusteredConnectorRequestReplicator.java | 87 +
.../ConnectorRequestReplicatorConfiguration.java | 46 +
.../FrameworkClusterConfiguration.java | 9 +-
.../TestThreadPoolRequestReplicator.java | 6 +-
.../nifi/controller/StandardProcessorNode.java | 102 +-
.../nifi/controller/flow/AbstractFlowManager.java | 2 +-
.../controller/flowanalysis/FlowAnalysisUtil.java | 6 +-
.../repository/StandardProcessSession.java | 31 +-
...StandardControllerServiceInvocationHandler.java | 40 +-
.../service/StandardControllerServiceNode.java | 92 +-
.../service/StandardControllerServiceProvider.java | 4 +-
.../service/StandardControllerServiceResolver.java | 6 +-
.../nifi/events/VolatileBulletinRepository.java | 1 +
.../StandardVersionedComponentSynchronizer.java | 34 +-
.../apache/nifi/groups/StandardProcessGroup.java | 57 +-
...rLogObserver.java => ConnectorLogObserver.java} | 21 +-
.../apache/nifi/logging/ProcessorLogObserver.java | 2 +-
.../StandardParameterReferenceManager.java | 17 +-
.../nifi/processor/StandardValidationContext.java | 28 +-
.../StandardValidationContextFactory.java | 6 +-
.../flow/FlowAnalyzingRegistryClientNode.java | 13 +-
...pper.java => VersionedComponentFlowMapper.java} | 100 +-
.../nifi/remote/StandardRemoteProcessGroup.java | 12 +-
.../repository/StandardProcessSessionTest.java | 4 +
.../StandardControllerServiceResolverTest.java | 6 +-
.../flow/FlowAnalyzingRegistryClientNodeTest.java | 4 +-
....java => TestVersionedComponentFlowMapper.java} | 10 +-
.../connector/ConfigurationStepConfiguration.java} | 19 +-
.../connector/ConfigurationUpdateResult.java} | 20 +-
.../connector/ConnectorConfiguration.java | 57 +
.../nifi/components/connector/ConnectorNode.java | 211 ++
.../components/connector/ConnectorRepository.java | 95 +
.../ConnectorRepositoryInitializationContext.java} | 24 +-
.../connector/ConnectorRequestReplicator.java} | 20 +-
.../connector/ConnectorState.java} | 28 +-
.../connector/ConnectorStateTransition.java | 82 +
.../connector/FlowContextFactory.java} | 21 +-
.../FrameworkConnectorInitializationContext.java} | 18 +-
...workConnectorInitializationContextBuilder.java} | 26 +-
.../connector/FrameworkFlowContext.java} | 23 +-
.../MutableConnectorConfigurationContext.java | 58 +
.../connector/ParameterContextFacadeFactory.java} | 21 +-
.../connector/ProcessGroupFacadeFactory.java} | 22 +-
.../connector/ProcessGroupFactory.java} | 20 +-
.../DisabledServiceValidationResult.java | 14 +
.../org/apache/nifi/connectable/Connectable.java | 2 +
.../connectable/ConnectableFlowFileActivity.java | 70 +
.../FlowFileActivity.java} | 24 +-
.../nifi/connectable/FlowFileTransferCounts.java | 88 +
.../connectable/ProcessGroupFlowFileActivity.java | 149 ++
.../nifi/controller/AbstractComponentNode.java | 48 +-
.../org/apache/nifi/controller/AbstractPort.java | 10 +
.../org/apache/nifi/controller/ComponentNode.java | 3 +
.../apache/nifi/controller/ProcessScheduler.java | 15 +-
.../org/apache/nifi/controller/ProcessorNode.java | 17 +-
.../org/apache/nifi/controller/StandardFunnel.java | 9 +
.../nifi/controller/ValidationContextFactory.java | 4 +-
.../apache/nifi/controller/flow/FlowManager.java | 17 +
.../nifi/controller/flow/VersionedDataflow.java | 10 +
.../nifi/controller/queue/DropFlowFileRequest.java | 16 +
.../controller/service/ControllerServiceNode.java | 10 +-
.../service/ControllerServiceProvider.java | 4 +-
.../org/apache/nifi/events/BulletinFactory.java | 7 +
.../java/org/apache/nifi/groups/ProcessGroup.java | 26 +-
.../org/apache/nifi/groups/RemoteProcessGroup.java | 4 +-
.../nifi-framework/nifi-framework-core/pom.xml | 6 +
.../components/connector/ConnectorDetails.java} | 33 +-
.../connector/ConnectorParameterLookup.java | 176 ++
.../ConnectorValidationContextBridge.java | 134 ++
.../connector/DescribedValueProvider.java} | 18 +-
.../nifi/components/connector/GhostConnector.java | 128 ++
.../StandaloneConnectorRequestReplicator.java} | 30 +-
.../StandardConnectorConfigurationContext.java | 328 +++
.../StandardConnectorInitializationContext.java | 122 ++
.../connector/StandardConnectorNode.java | 805 ++++++++
.../connector/StandardConnectorPropertyValue.java | 101 +
...StandardConnectorRepoInitializationContext.java | 59 +
.../connector/StandardConnectorRepository.java | 187 ++
.../StandardConnectorStateTransition.java | 113 ++
.../StandardConnectorValidationContext.java | 113 ++
.../components/connector/StandardFlowContext.java | 119 ++
.../standalone/ComponentContextProvider.java | 41 +
...egalExecutionEngineStatelessGroupLifecycle.java | 46 +
.../standalone/StandaloneConnectionFacade.java | 49 +
.../StandaloneControllerServiceFacade.java | 172 ++
.../StandaloneControllerServiceLifecycle.java | 62 +
.../StandaloneParameterContextFacade.java | 208 ++
.../standalone/StandaloneProcessGroupFacade.java | 380 ++++
.../StandaloneProcessGroupLifecycle.java | 294 +++
.../standalone/StandaloneProcessorFacade.java | 180 ++
.../standalone/StandaloneProcessorLifecycle.java | 93 +
.../StandaloneStatelessGroupLifecycle.java | 51 +
.../apache/nifi/controller/ExtensionBuilder.java | 343 ++--
.../org/apache/nifi/controller/FlowController.java | 75 +-
.../flow/FlowControllerFlowContextFactory.java | 114 ++
.../FlowControllerProcessGroupFacadeFactory.java | 70 +
.../flow/StandardComponentContextProvider.java | 105 +
.../nifi/controller/flow/StandardFlowManager.java | 151 +-
.../flow/StandardStatelessGroupNodeFactory.java | 4 +-
.../flowanalysis/StandardFlowAnalysisContext.java | 4 +-
.../scheduling/StandardProcessScheduler.java | 89 +-
.../serialization/VersionedDataflowMapper.java | 22 +-
.../serialization/VersionedFlowSynchronizer.java | 184 ++
.../VersionedReportingTaskSnapshotMapper.java | 6 +-
.../nifi/flowanalysis/StandardFlowAnalyzer.java | 10 +-
.../configuration/FlowControllerConfiguration.java | 78 +-
.../nifi/groups/StandardStatelessGroupNode.java | 9 +
....nifi.components.connector.ConnectorRepository} | 5 +-
.../components/connector/BlockingConnector.java | 126 ++
.../connector/DynamicAllowableValuesConnector.java | 137 ++
.../components/connector/DynamicFlowConnector.java | 335 ++++
.../components/connector/ParameterConnector.java | 130 ++
.../components/connector/SleepingConnector.java | 120 ++
.../connector/StandardConnectorNodeIT.java | 540 +++++
.../TestStandardConnectorConfigurationContext.java | 299 +++
...TestStandardConnectorInitializationContext.java | 128 ++
.../connector/TestStandardConnectorNode.java | 638 ++++++
.../connector/TestStandardConnectorRepository.java | 122 ++
.../connector/processors/CreateDummyFlowFile.java | 84 +
.../connector/processors/DuplicateFlowFile.java | 128 ++
.../connector/processors/ExposeFileValues.java | 72 +
.../connector/processors/LogFlowFileContents.java | 58 +
.../connector/processors/OverwriteFlowFile.java | 79 +-
.../components/connector/processors/Sleep.java | 151 ++
.../connector/processors/TerminateFlowFile.java} | 31 +-
.../connector/services/CounterService.java} | 20 +-
.../services/impl/StandardCounterService.java} | 27 +-
.../nifi/controller/MockStateManagerProvider.java | 70 +
.../apache/nifi/controller/flow/NopConnector.java | 162 ++
.../controller/flow/TestStandardFlowManager.java | 166 ++
.../TestStandardControllerServiceProvider.java | 41 +-
.../controller/service/mock/MockProcessGroup.java | 19 +
....java => VersionedComponentFlowMapperTest.java} | 4 +-
...org.apache.nifi.components.connector.Connector} | 7 +-
.../org.apache.nifi.controller.ControllerService | 5 +-
.../services/org.apache.nifi.processor.Processor | 9 +-
.../src/test/resources/flows/choose-color.json | 76 +
.../flows/generate-and-log-with-parameter.json | 253 +++
.../flows/generate-duplicate-log-flow.json | 690 +++++++
.../nar/StandardExtensionDiscoveringManager.java | 20 +-
.../java/org/apache/nifi/nar/DummyConnector.java | 114 ++
...rdExtensionDiscoveringManagerConnectorTest.java | 49 +
...org.apache.nifi.components.connector.Connector} | 7 +-
.../apache/nifi/headless/HeadlessNiFiServer.java | 36 +-
.../main/java/org/apache/nifi/nar/NarUnpacker.java | 4 +-
.../web/server/StandardServerProviderTest.java | 8 +-
.../nifi/authorization/AuthorizableLookup.java | 6 +
.../authorization/StandardAuthorizableLookup.java | 7 +
.../apache/nifi/web/StandardNiFiServiceFacade.java | 38 +-
.../web/dao/impl/StandardControllerServiceDAO.java | 2 +-
.../nifi/web/dao/impl/StandardProcessGroupDAO.java | 4 +-
.../nifi/web/dao/impl/StandardProcessorDAO.java | 3 +-
.../nifi/web/StandardNiFiServiceFacadeTest.java | 10 +-
.../nifi/web/api/TestDataTransferResource.java | 18 +-
.../dao/impl/TestStandardParameterContextDAO.java | 2 +-
.../StandardClientRegistrationProviderTest.java | 2 +-
.../StandardRegistrationBuilderProviderTest.java | 4 +-
...dardRelyingPartyRegistrationRepositoryTest.java | 4 +-
.../x509/X509AuthenticationProviderTest.java | 2 +-
.../java/org/apache/nifi/nar/ExtensionMapping.java | 25 +-
.../stateless/bootstrap/AllowListClassLoader.java | 5 +
.../stateless/bootstrap/StatelessBootstrap.java | 2 +-
.../scheduling/StatelessProcessScheduler.java | 29 +-
.../stateless/engine/StatelessFlowManager.java | 16 +
pom.xml | 4 +-
232 files changed, 19206 insertions(+), 973 deletions(-)
diff --git a/.cursorrules b/.cursorrules
new file mode 100644
index 0000000000..3bcdd1f6cd
--- /dev/null
+++ b/.cursorrules
@@ -0,0 +1,71 @@
+AI persona
+----------
+Act as an experienced Java software engineer. When considering how to implement a task, first consider the big picture of what is being asked. Then determine which classes will need to be udpated.
+Quite often, a single request will require manipulating many different classes. Generally speaking, it is best to avoid changing established interfaces, but it is often acceptable because the public API
+that NiFi adheres to is not in this module.
+
+
+API
+---
+When it is necessary to lookup the API for reference, the API can generally be found at ../nifi-api, where .. is relative to the project's root directory.
+
+Building
+--------
+NiFi is a complex Maven codebase. Never build code (testing or otherwise) using javac. Always use `mvn` instead.
+
+Code Style
+----------
+NiFi adheres to a few code styles that are not necessarily common. Please ensure that you observe these code styles.
+1. Any variable that can be marked `final` must be marked `final`. This includes declarations of Exceptions, method arguments, local variables, member variables, etc.
+2. Short-hand is highly discouraged in names of variables, classes, methods, etc., as well as in documentation.
+3. Private / helper methods should not be placed before the first public/protected method that calls it.
+4. Unless the method is to be heavily reused, avoid creating trivial 1-2 line methods and instead just place the code inline.
+5. Code is allowed to be up to 200 characters wide. Avoid breaking lines into many short lines.
+6. Avoid creating private methods that are called only once unless they are at least 10 lines long or are complex.
+7. It is never acceptable to use star imports. Import each individual class that is to be used.
+8. Never use underscores in class names, variables, or filenames.
+9. Never use System.out.println but instead use SLF4J Loggers
+10. Avoid excessive whitespace in method invocations. For example, instead of writing:
+```
+myObject.doSomething(
+ arg1,
+ arg2,
+ arg3,
+ arg4,
+ arg5
+);
+```
+Write this instead:
+```
+myObject.doSomething(arg1, arg2, arg3, arg4, arg5);
+```
+
+11. When possible, prefer importing a class, rather than using fully qualified classname inline in the code.
+12. Avoid statically importing methods unless they are used frequently such as methods in the `Assertions` and `Mockito` classes.
+13. Avoid trailing whitespace at the end of lines, especially in blank lines.
+
+Unit / Automated Testing
+------------------------
+In addition to the general rules defined in the Code Style section, follow these rules when creating or manipulating automated tests.
+1. NEVER add comments such as `// Given`, `// When`, `// Then`. These comments are considered an anti-pattern and should be removed or replaced whenever they are encountered.
+ Instead, leave them out all together (preferred) or use comments that clearly articulate what is happening, such as `// Setup`, `// Invoke method`, `// Assert expected results.`
+ One of the reasons that this is considered an anti-pattern (in addition to the fact that the given/when/then nomeclature itself provides no meaning) is that it assumes a very specific pattern
+ in unit tests, that we will create a bunch of objects, invoke the method we care about, make assertions, and then end. This often results in many tests that are extremely repetitive.
+ Instead, whenever it makes sense to do so, create the prerequisitive objects, invoke the method we care about with appropriate arguments, make assertions, and then invoke again with a different
+ set of arguments, make assertions, etc. There is no need to have many repetitive methods that each create many repetitive objects.
+
+2. Unit tests are Java. They are not English. As such, they should be written like Java. Frameworks such as assertj that strive to make the unit tests look more "English-like" should be avoided.
+ Use of these frameworks sometimes works well but often quickly devolves into automated tests that read like neither English nor Java.
+
+3. Like any other code, unit tests should be created using reusable methods where appropriate. Do not create 15 methods that are all very similar and repetitive. Instead, create reusable methods
+ that can be called from each of the methods.
+
+4. Never use the `assert` keyword. Use JUnit assertions instead.
+
+5. Never create a test file that is centered around testing a method or capability. Unit tests must always be named after the class they are testing. It is okay if a given unit test class is very, very long.
+
+6. This is a Java project using the Maven structure. Java test files must always fall under src/test/java of the appropriate sub-module.
+
+7. Never use pointless assertions such as assertDoesNotThrow - this adds nothing but complexity. Just call the method, and if it throws an Exception, the test will fail. I.e.,
+ it is assumed by default that each line does not throw an Exception, so do not use
+
diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java
index ad23f6204a..6ba9ea9221 100644
--- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java
+++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultFlowStateStrategyTest.java
@@ -111,13 +111,13 @@ public class DefaultFlowStateStrategyTest {
ProcessGroup rootProcessGroup = mock(ProcessGroup.class);
RemoteProcessGroup remoteProcessGroup = mock(RemoteProcessGroup.class);
Set<RemoteProcessGroup> remoteProcessGroups = Set.of(remoteProcessGroup);
- CompletableFuture<Void> rootProcessGroupStopFuture = mock(CompletableFuture.class);
- Future remoteProcessGroupStopFuture = mock(Future.class);
+ CompletableFuture<Void> rootProcessGroupStopFuture = CompletableFuture.completedFuture(null);
+ CompletableFuture<Void> remoteProcessGroupStopFuture = CompletableFuture.completedFuture(null);
ProcessGroup nestedProcessGroup = mock(ProcessGroup.class);
RemoteProcessGroup nestedRemoteProcessGroup = mock(RemoteProcessGroup.class);
Set<RemoteProcessGroup> nestedRemoteProcessGroups = Set.of(nestedRemoteProcessGroup);
- CompletableFuture<Void> nestedProcessGroupStopFuture = mock(CompletableFuture.class);
- Future nestedRemoteProcessGroupStopFuture = mock(Future.class);
+ CompletableFuture<Void> nestedProcessGroupStopFuture = CompletableFuture.completedFuture(null);
+ CompletableFuture<Void> nestedRemoteProcessGroupStopFuture = CompletableFuture.completedFuture(null);
when(flowManager.getRootGroup()).thenReturn(rootProcessGroup);
when(rootProcessGroup.getRemoteProcessGroups()).thenReturn(remoteProcessGroups);
diff --git a/nifi-commons/nifi-connector-utils/pom.xml b/nifi-commons/nifi-connector-utils/pom.xml
new file mode 100644
index 0000000000..2794ecbfd6
--- /dev/null
+++ b/nifi-commons/nifi-connector-utils/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-utils</artifactId>
+ <name>nifi-connector-utils</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/nifi-commons/nifi-connector-utils/src/main/java/org/apache/nifi/components/connector/util/VersionedFlowUtils.java b/nifi-commons/nifi-connector-utils/src/main/java/org/apache/nifi/components/connector/util/VersionedFlowUtils.java
new file mode 100644
index 0000000000..8eb839a3ba
--- /dev/null
+++ b/nifi-commons/nifi-connector-utils/src/main/java/org/apache/nifi/components/connector/util/VersionedFlowUtils.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.components.connector.util;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.nifi.flow.Bundle;
+import org.apache.nifi.flow.ComponentType;
+import org.apache.nifi.flow.ConnectableComponent;
+import org.apache.nifi.flow.ConnectableComponentType;
+import org.apache.nifi.flow.Position;
+import org.apache.nifi.flow.ScheduledState;
+import org.apache.nifi.flow.VersionedConnection;
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.flow.VersionedExternalFlow;
+import org.apache.nifi.flow.VersionedParameter;
+import org.apache.nifi.flow.VersionedParameterContext;
+import org.apache.nifi.flow.VersionedPort;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.flow.VersionedProcessor;
+
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Predicate;
+
+public class VersionedFlowUtils {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ public static VersionedExternalFlow loadFlowFromResource(final String resourceName) {
+ try (final InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName)) {
+ if (in == null) {
+ throw new IllegalArgumentException("Resource not found: " + resourceName);
+ }
+
+ return OBJECT_MAPPER.readValue(in, VersionedExternalFlow.class);
+ } catch (final Exception e) {
+ throw new IllegalStateException("Unable to load resource: " + resourceName, e);
+ }
+ }
+
+ public static Optional<VersionedProcessor> findProcessor(final VersionedProcessGroup group, final Predicate<VersionedProcessor> predicate) {
+ final List<VersionedProcessor> processors = findProcessors(group, predicate);
+ if (processors.size() == 1) {
+ return Optional.of(processors.getFirst());
+ }
+ return Optional.empty();
+ }
+
+ public static List<VersionedProcessor> findProcessors(final VersionedProcessGroup group, final Predicate<VersionedProcessor> predicate) {
+ final List<VersionedProcessor> processors = new ArrayList<>();
+ findProcessors(group, predicate, processors);
+ return processors;
+ }
+
+ private static void findProcessors(final VersionedProcessGroup group, final Predicate<VersionedProcessor> predicate, final List<VersionedProcessor> processors) {
+ for (final VersionedProcessor processor : group.getProcessors()) {
+ if (predicate.test(processor)) {
+ processors.add(processor);
+ }
+ }
+
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ findProcessors(childGroup, predicate, processors);
+ }
+ }
+
+ public static ConnectableComponent createConnectableComponent(final VersionedProcessor processor) {
+ final ConnectableComponent component = new ConnectableComponent();
+ component.setId(processor.getIdentifier());
+ component.setName(processor.getName());
+ component.setType(ConnectableComponentType.PROCESSOR);
+ component.setGroupId(processor.getGroupIdentifier());
+ return component;
+ }
+
+ public static ConnectableComponent createConnectableComponent(final VersionedPort port) {
+ final ConnectableComponent component = new ConnectableComponent();
+ component.setId(port.getIdentifier());
+ component.setName(port.getName());
+ component.setType(port.getComponentType() == ComponentType.INPUT_PORT ? ConnectableComponentType.INPUT_PORT : ConnectableComponentType.OUTPUT_PORT);
+ component.setGroupId(port.getGroupIdentifier());
+ return component;
+ }
+
+ public static void addConnection(final VersionedProcessGroup group, final ConnectableComponent source, final ConnectableComponent destination, final Set<String> relationships) {
+ final VersionedConnection connection = new VersionedConnection();
+ connection.setSource(source);
+ connection.setDestination(destination);
+ connection.setSelectedRelationships(relationships);
+ connection.setBends(List.of());
+ connection.setLabelIndex(0);
+ connection.setzIndex(0L);
+ connection.setGroupIdentifier(group.getIdentifier());
+ connection.setLoadBalanceStrategy("DO_NOT_LOAD_BALANCE");
+ connection.setBackPressureDataSizeThreshold("1 GB");
+ connection.setBackPressureObjectThreshold(10000L);
+ connection.setFlowFileExpiration("0 sec");
+ connection.setPrioritizers(new ArrayList<>());
+ connection.setComponentType(ComponentType.CONNECTION);
+
+ Set<VersionedConnection> connections = group.getConnections();
+ if (connections == null) {
+ connections = new HashSet<>();
+ group.setConnections(connections);
+ }
+ connections.add(connection);
+
+ final String uuid = generateDeterministicUuid(group, ComponentType.CONNECTION);
+ connection.setIdentifier(uuid);
+ }
+
+ public static List<VersionedConnection> findOutboundConnections(final VersionedProcessGroup group, final VersionedProcessor processor) {
+ final VersionedProcessGroup processorGroup = findGroupForProcessor(group, processor);
+ if (processorGroup == null) {
+ return List.of();
+ }
+
+ final List<VersionedConnection> outboundConnections = new ArrayList<>();
+ final Set<VersionedConnection> connections = processorGroup.getConnections();
+ if (connections == null) {
+ return outboundConnections;
+ }
+
+ for (final VersionedConnection connection : connections) {
+ final ConnectableComponent source = connection.getSource();
+ if (Objects.equals(source.getId(), processor.getIdentifier()) && source.getType() == ConnectableComponentType.PROCESSOR) {
+ outboundConnections.add(connection);
+ }
+ }
+
+ return outboundConnections;
+ }
+
+ public static VersionedProcessGroup findGroupForProcessor(final VersionedProcessGroup rootGroup, final VersionedProcessor processor) {
+ if (rootGroup.getProcessors().contains(processor)) {
+ return rootGroup;
+ }
+
+ for (final VersionedProcessGroup childGroup : rootGroup.getProcessGroups()) {
+ final VersionedProcessGroup foundGroup = findGroupForProcessor(childGroup, processor);
+ if (foundGroup != null) {
+ return foundGroup;
+ }
+ }
+
+ return null;
+ }
+
+ public static String generateDeterministicUuid(final VersionedProcessGroup group, final ComponentType componentType) {
+ final int componentCount = getComponentCount(group, componentType);
+ final String uuidSeed = "%s-%s-%d".formatted(group.getIdentifier(), componentType.name(), componentCount);
+ return UUID.nameUUIDFromBytes(uuidSeed.getBytes(StandardCharsets.UTF_8)).toString();
+ }
+
+ private static int getComponentCount(final VersionedProcessGroup group, final ComponentType componentType) {
+ final Collection<?> components = switch (componentType) {
+ case PROCESSOR -> group.getProcessors();
+ case INPUT_PORT -> group.getInputPorts();
+ case OUTPUT_PORT -> group.getOutputPorts();
+ case CONNECTION -> group.getConnections();
+ case FUNNEL -> group.getFunnels();
+ case LABEL -> group.getLabels();
+ case PROCESS_GROUP -> group.getProcessGroups();
+ case CONTROLLER_SERVICE -> group.getControllerServices();
+ default -> List.of();
+ };
+
+ return components == null ? 0 : components.size();
+ }
+
+ public static VersionedProcessor addProcessor(final VersionedProcessGroup group, final String processorType, final Bundle bundle, final String name, final Position position) {
+ final VersionedProcessor processor = new VersionedProcessor();
+
+ // Generate deterministic UUID based on group and component type
+ processor.setIdentifier(generateDeterministicUuid(group, ComponentType.PROCESSOR));
+
+ processor.setName(name);
+ processor.setType(processorType);
+ processor.setPosition(position);
+ processor.setBundle(bundle);
+
+ // Set default processor configuration
+ processor.setProperties(new HashMap<>());
+ processor.setPropertyDescriptors(new HashMap<>());
+ processor.setStyle(new HashMap<>());
+ processor.setSchedulingPeriod("0 sec");
+ processor.setSchedulingStrategy("TIMER_DRIVEN");
+ processor.setExecutionNode("ALL");
+ processor.setPenaltyDuration("30 sec");
+ processor.setYieldDuration("1 sec");
+ processor.setBulletinLevel("WARN");
+ processor.setRunDurationMillis(25L);
+ processor.setConcurrentlySchedulableTaskCount(1);
+ processor.setAutoTerminatedRelationships(new HashSet<>());
+ processor.setScheduledState(ScheduledState.ENABLED);
+ processor.setRetryCount(10);
+ processor.setRetriedRelationships(new HashSet<>());
+ processor.setBackoffMechanism("PENALIZE_FLOWFILE");
+ processor.setMaxBackoffPeriod("10 mins");
+ processor.setComponentType(ComponentType.PROCESSOR);
+ processor.setGroupIdentifier(group.getIdentifier());
+
+ group.getProcessors().add(processor);
+ return processor;
+ }
+
+ public static VersionedControllerService addControllerService(final VersionedProcessGroup group, final String serviceType, final Bundle bundle, final String name) {
+ final VersionedControllerService controllerService = new VersionedControllerService();
+
+ // Generate deterministic UUID based on group and component type
+ controllerService.setIdentifier(generateDeterministicUuid(group, ComponentType.CONTROLLER_SERVICE));
+
+ controllerService.setName(name);
+ controllerService.setType(serviceType);
+ controllerService.setBundle(bundle);
+ controllerService.setComponentType(ComponentType.CONTROLLER_SERVICE);
+
+ // Set default controller service configuration
+ controllerService.setProperties(new HashMap<>());
+ controllerService.setPropertyDescriptors(new HashMap<>());
+ controllerService.setControllerServiceApis(new ArrayList<>());
+ controllerService.setAnnotationData(null);
+ controllerService.setScheduledState(ScheduledState.DISABLED);
+ controllerService.setBulletinLevel("WARN");
+ controllerService.setComments(null);
+ controllerService.setGroupIdentifier(group.getIdentifier());
+
+ // Initialize controller services collection if it doesn't exist
+ Set<VersionedControllerService> controllerServices = group.getControllerServices();
+ if (controllerServices == null) {
+ controllerServices = new HashSet<>();
+ group.setControllerServices(controllerServices);
+ }
+ controllerServices.add(controllerService);
+
+ return controllerService;
+ }
+
+ public static Set<VersionedControllerService> getReferencedControllerServices(final VersionedProcessGroup group) {
+ final Set<VersionedControllerService> referencedServices = new HashSet<>();
+ collectReferencedControllerServices(group, referencedServices);
+ return referencedServices;
+ }
+
+ private static void collectReferencedControllerServices(final VersionedProcessGroup group, final Set<VersionedControllerService> referencedServices) {
+ final Map<String, VersionedControllerService> serviceMap = new HashMap<>();
+ for (final VersionedControllerService service : group.getControllerServices()) {
+ serviceMap.put(service.getIdentifier(), service);
+ }
+
+ for (final VersionedProcessor processor : group.getProcessors()) {
+ for (final String propertyValue : processor.getProperties().values()) {
+ final VersionedControllerService referencedService = serviceMap.get(propertyValue);
+ if (referencedService != null) {
+ referencedServices.add(referencedService);
+ }
+ }
+ }
+
+ while (true) {
+ final Set<VersionedControllerService> newlyAddedServices = new HashSet<>();
+
+ for (final VersionedControllerService service : referencedServices) {
+ for (final String propertyValue : service.getProperties().values()) {
+ final VersionedControllerService referencedService = serviceMap.get(propertyValue);
+ if (referencedService != null && !referencedServices.contains(referencedService)) {
+ newlyAddedServices.add(referencedService);
+ }
+ }
+ }
+
+ referencedServices.addAll(newlyAddedServices);
+ if (newlyAddedServices.isEmpty()) {
+ break;
+ }
+ }
+
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ collectReferencedControllerServices(childGroup, referencedServices);
+ }
+ }
+
+ public static void removeControllerServiceReferences(final VersionedProcessGroup processGroup, final String serviceIdentifier) {
+ for (final VersionedProcessor processor : processGroup.getProcessors()) {
+ removeValuesFromMap(processor.getProperties(), serviceIdentifier);
+ }
+
+ for (final VersionedControllerService service : processGroup.getControllerServices()) {
+ removeValuesFromMap(service.getProperties(), serviceIdentifier);
+ }
+
+ for (final VersionedProcessGroup childGroup : processGroup.getProcessGroups()) {
+ removeControllerServiceReferences(childGroup, serviceIdentifier);
+ }
+ }
+
+ private static void removeValuesFromMap(final Map<String, String> properties, final String valueToRemove) {
+ final List<String> keysToRemove = new ArrayList<>();
+ for (final Map.Entry<String, String> entry : properties.entrySet()) {
+ if (Objects.equals(entry.getValue(), valueToRemove)) {
+ keysToRemove.add(entry.getKey());
+ }
+ }
+
+ keysToRemove.forEach(properties::remove);
+ }
+
+ public static void setParameterValue(final VersionedExternalFlow externalFlow, final String parameterName, final String parameterValue) {
+ for (final VersionedParameterContext context : externalFlow.getParameterContexts().values()) {
+ setParameterValue(context, parameterName, parameterValue);
+ }
+ }
+
+ public static void setParameterValue(final VersionedParameterContext parameterContext, final String parameterName, final String parameterValue) {
+ final Set<VersionedParameter> parameters = parameterContext.getParameters();
+ for (final VersionedParameter parameter : parameters) {
+ if (parameter.getName().equals(parameterName)) {
+ parameter.setValue(parameterValue);
+ }
+ }
+ }
+
+ public static void removeUnreferencedControllerServices(final VersionedProcessGroup processGroup) {
+ final Set<VersionedControllerService> referencedServices = getReferencedControllerServices(processGroup);
+ final Set<String> referencedServiceIds = new HashSet<>();
+ for (final VersionedControllerService service : referencedServices) {
+ referencedServiceIds.add(service.getIdentifier());
+ }
+
+ removeUnreferencedControllerServices(processGroup, referencedServiceIds);
+ }
+
+ private static void removeUnreferencedControllerServices(final VersionedProcessGroup processGroup, final Set<String> referencedServiceIds) {
+ processGroup.getControllerServices().removeIf(service -> !referencedServiceIds.contains(service.getIdentifier()));
+
+ for (final VersionedProcessGroup childGroup : processGroup.getProcessGroups()) {
+ removeUnreferencedControllerServices(childGroup, referencedServiceIds);
+ }
+ }
+}
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index 86e83526ce..e70e34036b 100644
--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -136,6 +136,9 @@ public class NiFiProperties extends ApplicationProperties {
public static final String STATUS_REPOSITORY_QUESTDB_PERSIST_BATCH_SIZE = "nifi.status.repository.questdb.persist.batchsize";
public static final String STATUS_REPOSITORY_QUESTDB_PERSIST_FREQUENCY = "nifi.status.repository.questdb.persist.frequency";
+ // Connector Repository properties
+ public static final String CONNECTOR_REPOSITORY_IMPLEMENTATION = "nifi.components.connectors.repository.implementation";
+
// security properties
public static final String SECURITY_KEYSTORE = "nifi.security.keystore";
public static final String SECURITY_KEYSTORE_TYPE = "nifi.security.keystoreType";
@@ -1811,8 +1814,28 @@ public class NiFiProperties extends ApplicationProperties {
// The Properties(Properties) constructor does NOT inherit the provided values, just uses them as default values
if (additionalProperties != null) {
- additionalProperties.forEach(properties::put);
+ properties.putAll(additionalProperties);
+ }
+
+ return createNiFiProperties(properties);
+ }
+
+ public static NiFiProperties createBasicNiFiProperties(final InputStream inputStream) {
+ return createBasicNiFiProperties(inputStream, null);
+ }
+
+ public static NiFiProperties createBasicNiFiProperties(final InputStream inputStream, final Properties additionalProperties) {
+ final Properties properties = new Properties();
+ readFromInputStream(inputStream, properties);
+
+ if (additionalProperties != null) {
+ properties.putAll(additionalProperties);
}
+
+ return createNiFiProperties(properties);
+ }
+
+ private static NiFiProperties createNiFiProperties(final Properties properties) {
return new NiFiProperties() {
@Override
public String getProperty(String key) {
@@ -1832,9 +1855,7 @@ public class NiFiProperties extends ApplicationProperties {
}
private static void readFromPropertiesFile(String propertiesFilePath, Properties properties) {
- final String nfPropertiesFilePath = (propertiesFilePath == null)
- ? System.getProperty(PROPERTIES_FILE_PATH)
- : propertiesFilePath;
+ final String nfPropertiesFilePath = (propertiesFilePath == null) ? System.getProperty(PROPERTIES_FILE_PATH) : propertiesFilePath;
if (nfPropertiesFilePath != null) {
final File propertiesFile = new File(nfPropertiesFilePath.trim());
if (!propertiesFile.exists()) {
@@ -1845,24 +1866,25 @@ public class NiFiProperties extends ApplicationProperties {
throw new RuntimeException("Properties file exists but cannot be read '"
+ propertiesFile.getAbsolutePath() + "'");
}
- InputStream inStream = null;
- try {
- inStream = new BufferedInputStream(new FileInputStream(propertiesFile));
+
+ try (final InputStream fileIn = new FileInputStream(propertiesFile);
+ final InputStream inStream = new BufferedInputStream(fileIn)) {
+
properties.load(inStream);
} catch (final Exception ex) {
- throw new RuntimeException("Cannot load properties file due to "
- + ex.getLocalizedMessage(), ex);
- } finally {
- if (null != inStream) {
- try {
- inStream.close();
- } catch (final Exception ignored) {
- }
- }
+ throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
}
}
}
+ private static void readFromInputStream(final InputStream inputStream, final Properties properties) {
+ try {
+ properties.load(inputStream);
+ } catch (final Exception ex) {
+ throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
+ }
+ }
+
/**
* This method is used to validate the NiFi properties when the file is loaded
* for the first time. The objective is to stop NiFi startup in case a property
diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml
index 53179adcac..a4173dfe0a 100644
--- a/nifi-commons/pom.xml
+++ b/nifi-commons/pom.xml
@@ -68,6 +68,7 @@
<module>nifi-write-ahead-log</module>
<module>nifi-xml-processing</module>
<module>nifi-calcite-utils</module>
+ <module>nifi-connector-utils</module>
</modules>
<properties>
<guava.version>33.5.0-jre</guava.version>
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-api/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-api/pom.xml
new file mode 100644
index 0000000000..2d581f17da
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-server-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorConfigVerificationResult.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorConfigVerificationResult.java
index 96806fca1d..9ec3ac2878 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorConfigVerificationResult.java
@@ -14,23 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.components.ConfigVerificationResult;
-public interface ValidationContextFactory {
+import java.util.List;
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+public interface ConnectorConfigVerificationResult {
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ List<ConfigVerificationResult> getAllResults();
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ List<ConfigVerificationResult> getFailedResults();
+
+ void assertNoFailures();
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorMockServer.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorMockServer.java
index 96806fca1d..7b24c973c9 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorMockServer.java
@@ -14,23 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.NiFiServer;
+import org.apache.nifi.processor.Processor;
-public interface ValidationContextFactory {
+import java.io.File;
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+public interface ConnectorMockServer extends NiFiServer, ConnectorTestRunner {
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ void instantiateConnector(String connectorClassName);
+
+ void registerMockBundle(ClassLoader classLoader, File workingDirectory);
+
+ void mockProcessor(String processorType, Class<? extends Processor> mockProcessorClass);
- Optional<FlowAnalyzer> getFlowAnalyzer();
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java
new file mode 100644
index 0000000000..2c3eaa40ee
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.processor.Processor;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.List;
+
+public interface ConnectorTestRunner extends Closeable {
+
+ void applyUpdate() throws FlowUpdateException;
+
+ default void configure(String stepName, PropertyGroupConfiguration groupConfiguration) throws FlowUpdateException {
+ configure(stepName, List.of(groupConfiguration));
+ }
+
+ void configure(String stepName, List<PropertyGroupConfiguration> groupConfigurations) throws FlowUpdateException;
+
+ ConnectorConfigVerificationResult verifyConfiguration(String stepName, List<PropertyGroupConfiguration> groupConfigurations);
+
+ void startConnector();
+
+ void stopConnector();
+
+ void waitForDataIngested(Duration maxWaitTime);
+
+ void waitForIdle(Duration maxWaitTime);
+
+ void waitForIdle(Duration minimumIdleTime, Duration maxWaitTime);
+
+ List<ValidationResult> validate();
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml
new file mode 100644
index 0000000000..eb4b1e8f72
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-server-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-server</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <repositories>
+ <!-- Shibboleth Repository required for OpenSAML -->
+ <!-- This is required because nifi-framework-nar is the parent/NAR Dependency, andi t requires shibboleth. -->
+ <repository>
+ <id>shibboleth</id>
+ <url>https://build.shibboleth.net/nexus/content/repositories/releases/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+</project>
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-server/pom.xml
new file mode 100644
index 0000000000..040596e620
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-server</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-core-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-core</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-encryptor</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-server-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java
new file mode 100644
index 0000000000..3aa7bbc543
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.service.AuditService;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.HistoryQuery;
+import org.apache.nifi.history.PreviousValue;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class MockAuditService implements AuditService {
+ @Override
+ public void addActions(final Collection<Action> actions) {
+ }
+
+ @Override
+ public Map<String, List<PreviousValue>> getPreviousValues(final String componentId) {
+ return Map.of();
+ }
+
+ @Override
+ public void deletePreviousValues(final String propertyName, final String componentId) {
+ }
+
+ @Override
+ public History getActions(final HistoryQuery actionQuery) {
+ return null;
+ }
+
+ @Override
+ public History getActions(final int firstActionId, final int maxActions) {
+ return null;
+ }
+
+ @Override
+ public Action getAction(final Integer actionId) {
+ return null;
+ }
+
+ @Override
+ public void purgeActions(final Date end, final Action purgeAction) {
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorInitializationContext.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorInitializationContext.java
new file mode 100644
index 0000000000..38dcec3474
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorInitializationContext.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.asset.AssetManager;
+import org.apache.nifi.components.connector.FrameworkConnectorInitializationContextBuilder;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.FrameworkConnectorInitializationContext;
+import org.apache.nifi.components.connector.FrameworkFlowContext;
+import org.apache.nifi.components.connector.SecretsManager;
+import org.apache.nifi.components.connector.components.FlowContext;
+import org.apache.nifi.flow.VersionedExternalFlow;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.flow.VersionedProcessor;
+import org.apache.nifi.logging.ComponentLog;
+
+public class MockConnectorInitializationContext implements FrameworkConnectorInitializationContext {
+
+ private final String identifier;
+ private final String name;
+ private final ComponentLog componentLog;
+ private final SecretsManager secretsManager;
+ private final AssetManager assetManager;
+
+ private final MockExtensionMapper mockExtensionMapper;
+
+ private MockConnectorInitializationContext(final Builder builder) {
+ this.identifier = builder.identifier;
+ this.name = builder.name;
+ this.componentLog = builder.componentLog;
+ this.secretsManager = builder.secretsManager;
+ this.assetManager = builder.assetManager;
+ this.mockExtensionMapper = builder.mockExtensionMapper;
+ }
+
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public ComponentLog getLogger() {
+ return componentLog;
+ }
+
+ @Override
+ public SecretsManager getSecretsManager() {
+ return secretsManager;
+ }
+
+ @Override
+ public AssetManager getAssetManager() {
+ return assetManager;
+ }
+
+ @Override
+ public void updateFlow(final FlowContext flowContext, final VersionedExternalFlow versionedExternalFlow) throws FlowUpdateException {
+ if (!(flowContext instanceof final FrameworkFlowContext frameworkFlowContext)) {
+ throw new IllegalArgumentException("FlowContext is not an instance provided by the framework");
+ }
+
+ replaceMocks(versionedExternalFlow.getFlowContents());
+
+ // TODO: Probably should eliminate this method and instead move AssetManager to the FlowContext and add a method there
+ // to update the flow.
+ frameworkFlowContext.updateFlow(versionedExternalFlow, assetManager);
+ }
+
+ private void replaceMocks(final VersionedProcessGroup group) {
+ if (group.getProcessors() != null) {
+ for (final VersionedProcessor processor : group.getProcessors()) {
+ mockExtensionMapper.mapProcessor(processor);
+ }
+ }
+
+ if (group.getProcessGroups() != null) {
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ replaceMocks(childGroup);
+ }
+ }
+ }
+
+ private void updateParameterContext(final VersionedProcessGroup group, final String parameterContextName) {
+ group.setParameterContextName(parameterContextName);
+ if (group.getProcessGroups() != null) {
+ for (final VersionedProcessGroup childGroup : group.getProcessGroups()) {
+ updateParameterContext(childGroup, parameterContextName);
+ }
+ }
+ }
+
+
+ public static class Builder implements FrameworkConnectorInitializationContextBuilder {
+ private final MockExtensionMapper mockExtensionMapper;
+ private String identifier;
+ private String name;
+ private ComponentLog componentLog;
+ private SecretsManager secretsManager;
+ private AssetManager assetManager;
+
+ public Builder(final MockExtensionMapper mockExtensionMapper) {
+ this.mockExtensionMapper = mockExtensionMapper;
+ }
+
+ public Builder identifier(final String identifier) {
+ this.identifier = identifier;
+ return this;
+ }
+
+ public Builder name(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder componentLog(final ComponentLog componentLog) {
+ this.componentLog = componentLog;
+ return this;
+ }
+
+ public Builder secretsManager(final SecretsManager secretsManager) {
+ this.secretsManager = secretsManager;
+ return this;
+ }
+
+ public Builder assetManager(final AssetManager assetManager) {
+ this.assetManager = assetManager;
+ return this;
+ }
+
+ public MockConnectorInitializationContext build() {
+ return new MockConnectorInitializationContext(this);
+ }
+
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorRepository.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorRepository.java
index 96806fca1d..9f95111674 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorRepository.java
@@ -14,23 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.components.connector.FrameworkConnectorInitializationContextBuilder;
+import org.apache.nifi.components.connector.StandardConnectorRepository;
-public interface ValidationContextFactory {
+public class MockConnectorRepository extends StandardConnectorRepository {
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+ private volatile MockExtensionMapper mockExtensionMapper;
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ public void setMockExtensionMapper(final MockExtensionMapper mapper) {
+ mockExtensionMapper = mapper;
+ }
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ @Override
+ public FrameworkConnectorInitializationContextBuilder createInitializationContextBuilder() {
+ return new MockConnectorInitializationContext.Builder(mockExtensionMapper);
+ }
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java
new file mode 100644
index 0000000000..7af2240d2d
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.bundle.BundleCoordinate;
+import org.apache.nifi.bundle.BundleDetails;
+import org.apache.nifi.nar.InstanceClassLoader;
+import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
+import org.apache.nifi.processor.Processor;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class MockExtensionDiscoveringManager extends StandardExtensionDiscoveringManager {
+ private final ConcurrentMap<String, ClassLoader> mockComponentClassLoaders = new ConcurrentHashMap<>();
+
+ public synchronized void addProcessor(final Class<? extends Processor> mockProcessorClass) {
+ final BundleDetails bundleDetails = new BundleDetails.Builder()
+ .workingDir(new File("target/work/extensions/mock-bundle"))
+ .coordinate(new BundleCoordinate("org.apache.nifi.mock", mockProcessorClass.getName(), "1.0.0"))
+ .build();
+
+ final Bundle mockBundle = new Bundle(bundleDetails, mockProcessorClass.getClassLoader());
+ discoverExtensions(Set.of(mockBundle));
+
+ mockComponentClassLoaders.put(mockProcessorClass.getName(), mockProcessorClass.getClassLoader());
+ registerExtensionClass(Processor.class, mockProcessorClass.getName(), mockBundle);
+ }
+
+ @Override
+ public InstanceClassLoader createInstanceClassLoader(final String classType, final String instanceIdentifier, final Bundle bundle, final Set<URL> additionalUrls,
+ final boolean register, final String classloaderIsolationKey) {
+
+ final ClassLoader classLoader = mockComponentClassLoaders.get(classType);
+ if (classLoader != null) {
+ return new InstanceClassLoader(instanceIdentifier, classType, additionalUrls, Set.of(), classLoader);
+ }
+
+ return super.createInstanceClassLoader(classType, instanceIdentifier, bundle, additionalUrls, register, classloaderIsolationKey);
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java
new file mode 100644
index 0000000000..ed532cc253
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.flow.Bundle;
+import org.apache.nifi.flow.VersionedProcessor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MockExtensionMapper {
+
+ private final Map<String, String> processorMocks = new HashMap<>();
+
+ public void mockProcessor(final String processorType, final String mockProcessorClassName) {
+ processorMocks.put(processorType, mockProcessorClassName);
+ }
+
+ public void mapProcessor(final VersionedProcessor processor) {
+ final String type = processor.getType();
+ final String implementationClassName = processorMocks.get(type);
+ if (implementationClassName == null) {
+ return;
+ }
+
+ processor.setType(implementationClassName);
+ processor.setBundle(new Bundle("org.apache.nifi.mock", implementationClassName, "1.0.0"));
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockRuleViolationsManager.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockRuleViolationsManager.java
new file mode 100644
index 0000000000..4c3d5fd8c5
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockRuleViolationsManager.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.flow.VersionedComponent;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.validation.RuleViolation;
+import org.apache.nifi.validation.RuleViolationsManager;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class MockRuleViolationsManager implements RuleViolationsManager {
+ @Override
+ public void upsertComponentViolations(final String subjectId, final Collection<RuleViolation> violations) {
+
+ }
+
+ @Override
+ public void upsertGroupViolations(final VersionedProcessGroup processGroup, final Collection<RuleViolation> violations, final Map<VersionedComponent, Collection<RuleViolation>> componentToRuleViolations) {
+
+ }
+
+ @Override
+ public Collection<RuleViolation> getRuleViolationsForSubject(final String subjectId) {
+ return List.of();
+ }
+
+ @Override
+ public Collection<RuleViolation> getRuleViolationsForGroup(final String groupId) {
+ return List.of();
+ }
+
+ @Override
+ public Collection<RuleViolation> getRuleViolationsForGroups(final Collection<String> groupIds) {
+ return List.of();
+ }
+
+ @Override
+ public Collection<RuleViolation> getAllRuleViolations() {
+ return List.of();
+ }
+
+ @Override
+ public void removeRuleViolationsForSubject(final String subjectId) {
+
+ }
+
+ @Override
+ public void removeRuleViolationsForRule(final String ruleId) {
+
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockServerConfigVerificationResult.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockServerConfigVerificationResult.java
new file mode 100644
index 0000000000..fed82f9290
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockServerConfigVerificationResult.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
+
+import java.util.Collections;
+import java.util.List;
+
+public class MockServerConfigVerificationResult implements ConnectorConfigVerificationResult {
+ private final List<ConfigVerificationResult> results;
+
+ public MockServerConfigVerificationResult(final List<ConfigVerificationResult> results) {
+ this.results = results;
+ }
+
+ @Override
+ public List<ConfigVerificationResult> getAllResults() {
+ return Collections.unmodifiableList(results);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> getFailedResults() {
+ return results.stream()
+ .filter(result -> result.getOutcome() == Outcome.FAILED)
+ .toList();
+ }
+
+ @Override
+ public void assertNoFailures() {
+ final List<ConfigVerificationResult> failedResults = getFailedResults();
+ if (!failedResults.isEmpty()) {
+ throw new AssertionError("Configuration verification failed: " + failedResults);
+ }
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManager.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManager.java
new file mode 100644
index 0000000000..f22269ceab
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManager.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.Scope;
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.components.state.StateMap;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MockStateManager implements StateManager {
+ private final AtomicLong versionCounter = new AtomicLong(0L);
+ private final boolean dropStateKeySupported;
+
+ private volatile StateMap localStateMap = new MockStateMap(null, -1L);
+ private volatile StateMap clusterStateMap = new MockStateMap(null, -1L);
+
+ public MockStateManager(final boolean dropStateKeySupported) {
+ this.dropStateKeySupported = dropStateKeySupported;
+ }
+
+ @Override
+ public synchronized void setState(final Map<String, String> state, final Scope scope) throws IOException {
+ final StateMap newStateMap = new MockStateMap(state, versionCounter.incrementAndGet());
+
+ if (scope == Scope.CLUSTER) {
+ clusterStateMap = newStateMap;
+ } else {
+ localStateMap = newStateMap;
+ }
+ }
+
+ @Override
+ public synchronized StateMap getState(final Scope scope) throws IOException {
+ if (scope == Scope.CLUSTER) {
+ return clusterStateMap;
+ } else {
+ return localStateMap;
+ }
+ }
+
+ @Override
+ public synchronized boolean replace(final StateMap oldValue, final Map<String, String> newValue, final Scope scope) throws IOException {
+ final StateMap currentStateMap = scope == Scope.CLUSTER ? clusterStateMap : localStateMap;
+
+ if (currentStateMap == oldValue) {
+ final StateMap newStateMap = new MockStateMap(newValue, versionCounter.incrementAndGet());
+
+ if (scope == Scope.CLUSTER) {
+ clusterStateMap = newStateMap;
+ } else {
+ localStateMap = newStateMap;
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public synchronized void clear(final Scope scope) throws IOException {
+ setState(Collections.emptyMap(), scope);
+ }
+
+ @Override
+ public boolean isStateKeyDropSupported() {
+ return dropStateKeySupported;
+ }
+}
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManagerProvider.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManagerProvider.java
new file mode 100644
index 0000000000..77484904f2
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateManagerProvider.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.components.state.StateManagerProvider;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class MockStateManagerProvider implements StateManagerProvider {
+ private final ConcurrentMap<String, StateManager> stateManagers = new ConcurrentHashMap<>();
+ private volatile boolean clusterProviderEnabled = false;
+
+ @Override
+ public StateManager getStateManager(final String componentId, final boolean dropStateKeySupported) {
+ return stateManagers.computeIfAbsent(componentId, id -> new MockStateManager(dropStateKeySupported));
+ }
+
+ @Override
+ public void onComponentRemoved(final String componentId) {
+ stateManagers.remove(componentId);
+ }
+
+ @Override
+ public void shutdown() {
+ stateManagers.clear();
+ }
+
+ @Override
+ public void enableClusterProvider() {
+ clusterProviderEnabled = true;
+ }
+
+ @Override
+ public void disableClusterProvider() {
+ clusterProviderEnabled = false;
+ }
+
+ @Override
+ public boolean isClusterProviderEnabled() {
+ return clusterProviderEnabled;
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateMap.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateMap.java
new file mode 100644
index 0000000000..54b0980d61
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockStateMap.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.StateMap;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class MockStateMap implements StateMap {
+ private final Map<String, String> stateValues;
+ private final long version;
+
+ public MockStateMap(final Map<String, String> stateValues, final long version) {
+ this.stateValues = stateValues == null ? Collections.emptyMap() : new HashMap<>(stateValues);
+ this.version = version;
+ }
+
+ @Override
+ public Optional<String> getStateVersion() {
+ return version == -1L ? Optional.empty() : Optional.of(Long.toString(version));
+ }
+
+ @Override
+ public String get(final String key) {
+ return stateValues.get(key);
+ }
+
+ @Override
+ public Map<String, String> toMap() {
+ return Collections.unmodifiableMap(stateValues);
+ }
+
+ @Override
+ public String toString() {
+ return "MockStateMap[version=" + version + ", values=" + stateValues + "]";
+ }
+}
+
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/NopPropertyEncryptor.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/NopPropertyEncryptor.java
index 96806fca1d..56fbe816b7 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/NopPropertyEncryptor.java
@@ -14,23 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connector.server;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.encrypt.PropertyEncryptor;
-public interface ValidationContextFactory {
+public class NopPropertyEncryptor implements PropertyEncryptor {
+ @Override
+ public String encrypt(final String property) {
+ return property;
+ }
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
-
- Optional<RuleViolationsManager> getRuleViolationsManager();
-
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ @Override
+ public String decrypt(final String encryptedProperty) {
+ return encryptedProperty;
+ }
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/PermitAllAuthorizer.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/PermitAllAuthorizer.java
new file mode 100644
index 0000000000..75b2577f25
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/PermitAllAuthorizer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.authorization.AuthorizationRequest;
+import org.apache.nifi.authorization.AuthorizationResult;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.authorization.AuthorizerConfigurationContext;
+import org.apache.nifi.authorization.AuthorizerInitializationContext;
+import org.apache.nifi.authorization.exception.AuthorizationAccessException;
+import org.apache.nifi.authorization.exception.AuthorizerCreationException;
+import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
+
+public class PermitAllAuthorizer implements Authorizer {
+ @Override
+ public AuthorizationResult authorize(final AuthorizationRequest request) throws AuthorizationAccessException {
+ return AuthorizationResult.approved();
+ }
+
+ @Override
+ public void initialize(final AuthorizerInitializationContext initializationContext) throws AuthorizerCreationException {
+ }
+
+ @Override
+ public void onConfigured(final AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException {
+ }
+
+ @Override
+ public void preDestruction() throws AuthorizerDestructionException {
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/StandardConnectorMockServer.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/StandardConnectorMockServer.java
new file mode 100644
index 0000000000..1b8034c9ff
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/StandardConnectorMockServer.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.admin.service.AuditService;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.bundle.BundleCoordinate;
+import org.apache.nifi.bundle.BundleDetails;
+import org.apache.nifi.cluster.ClusterDetailsFactory;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.ConnectorNode;
+import org.apache.nifi.components.connector.ConnectorRepository;
+import org.apache.nifi.components.connector.ConnectorState;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.components.connector.StandaloneConnectorRequestReplicator;
+import org.apache.nifi.components.state.StateManagerProvider;
+import org.apache.nifi.components.validation.DisabledServiceValidationResult;
+import org.apache.nifi.components.validation.ValidationState;
+import org.apache.nifi.connectable.FlowFileTransferCounts;
+import org.apache.nifi.controller.DecommissionTask;
+import org.apache.nifi.controller.FlowController;
+import org.apache.nifi.controller.repository.FlowFileEventRepository;
+import org.apache.nifi.controller.repository.metrics.RingBufferEventRepository;
+import org.apache.nifi.controller.status.history.StatusHistoryDumpFactory;
+import org.apache.nifi.controller.status.history.StatusHistoryRepository;
+import org.apache.nifi.controller.status.history.VolatileComponentStatusRepository;
+import org.apache.nifi.diagnostics.DiagnosticsFactory;
+import org.apache.nifi.encrypt.PropertyEncryptor;
+import org.apache.nifi.engine.FlowEngine;
+import org.apache.nifi.events.VolatileBulletinRepository;
+import org.apache.nifi.nar.ExtensionMapping;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.reporting.BulletinRepository;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.validation.RuleViolationsManager;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public class StandardConnectorMockServer implements ConnectorMockServer {
+ private static final String CONNECTOR_ID = "test-connector";
+
+ private Bundle systemBundle;
+ private Set<Bundle> bundles;
+ private NiFiProperties nifiProperties;
+ private FlowController flowController;
+ private MockExtensionDiscoveringManager extensionManager;
+ private ConnectorNode connectorNode;
+ private FlowEngine flowEngine;
+ private MockExtensionMapper mockExtensionMapper;
+ private FlowFileTransferCounts initialFlowFileTransferCounts = new FlowFileTransferCounts(0L, 0L, 0L, 0L);
+
+ @Override
+ public void start() {
+ extensionManager = new MockExtensionDiscoveringManager();
+ extensionManager.discoverExtensions(systemBundle, bundles);
+ extensionManager.logClassLoaderMapping();
+
+ final FlowFileEventRepository flowFileEventRepository = new RingBufferEventRepository(5);
+ final Authorizer authorizer = new PermitAllAuthorizer();
+ final AuditService auditService = new MockAuditService();
+ final PropertyEncryptor propertyEncryptor = new NopPropertyEncryptor();
+ final BulletinRepository bulletinRepository = new VolatileBulletinRepository();
+ final StatusHistoryRepository statusHistoryRepository = new VolatileComponentStatusRepository(nifiProperties);
+ final RuleViolationsManager ruleViolationManager = new MockRuleViolationsManager();
+ final StateManagerProvider stateManagerProvider = new MockStateManagerProvider();
+
+ flowController = FlowController.createStandaloneInstance(
+ flowFileEventRepository,
+ null,
+ nifiProperties,
+ authorizer,
+ auditService,
+ propertyEncryptor,
+ bulletinRepository,
+ extensionManager,
+ statusHistoryRepository,
+ ruleViolationManager,
+ stateManagerProvider,
+ new StandaloneConnectorRequestReplicator());
+
+ try {
+ flowController.getRepositoryContextFactory().getFlowFileRepository().loadFlowFiles(Collections::emptyList);
+ } catch (final IOException e) {
+ throw new RuntimeException("Failed to initialize FlowFile Repository", e);
+ }
+
+ final ConnectorRepository connectorRepository = flowController.getConnectorRepository();
+ if (!(connectorRepository instanceof MockConnectorRepository)) {
+ throw new IllegalStateException("Connector Repository is not an instance of MockConnectorRepository");
+ }
+
+ mockExtensionMapper = new MockExtensionMapper();
+ ((MockConnectorRepository) connectorRepository).setMockExtensionMapper(mockExtensionMapper);
+
+ flowEngine = new FlowEngine(4, "Connector Threads");
+ }
+
+ @Override
+ public void initialize(final NiFiProperties properties, final Bundle systemBundle, final Set<Bundle> bundles, final ExtensionMapping extensionMapping) {
+ this.systemBundle = systemBundle;
+ this.bundles = bundles;
+ this.nifiProperties = properties;
+ }
+
+ @Override
+ public void stop() {
+ if (flowEngine != null) {
+ flowEngine.shutdown();
+ }
+ if (flowController != null) {
+ flowController.shutdown(false);
+ }
+ }
+
+ @Override
+ public DiagnosticsFactory getDiagnosticsFactory() {
+ return null;
+ }
+
+ @Override
+ public DiagnosticsFactory getThreadDumpFactory() {
+ return null;
+ }
+
+ @Override
+ public DecommissionTask getDecommissionTask() {
+ return null;
+ }
+
+ @Override
+ public ClusterDetailsFactory getClusterDetailsFactory() {
+ return null;
+ }
+
+ @Override
+ public StatusHistoryDumpFactory getStatusHistoryDumpFactory() {
+ return null;
+ }
+
+ @Override
+ public void instantiateConnector(final String connectorClassName) {
+ final List<Bundle> bundles = extensionManager.getBundles(connectorClassName);
+ if (bundles.isEmpty()) {
+ throw new IllegalStateException("No bundles found for connector class: " + connectorClassName + " - ensure that you have included all relevant NARs in the configured lib directory");
+ }
+ if (bundles.size() > 1) {
+ throw new IllegalStateException("Multiple bundles found for connector class: " + connectorClassName + " - unable to determine which bundle to use. Ensure that only a single version of " +
+ "the Connector is included in the configured lib directory. Available bundles: " + bundles);
+ }
+
+ final BundleCoordinate bundleCoordinate = bundles.getFirst().getBundleDetails().getCoordinate();
+ connectorNode = flowController.getFlowManager().createConnector(connectorClassName, CONNECTOR_ID, bundleCoordinate, true, true);
+ }
+
+ @Override
+ public void registerMockBundle(final ClassLoader classLoader, final File workingDirectory) {
+ final BundleDetails mockBundleDetails = new BundleDetails.Builder()
+ .workingDir(workingDirectory)
+ .coordinate(new BundleCoordinate("org.apache.nifi.mock", "nifi-connector-mock-bundle", "1.0.0"))
+ .build();
+ }
+
+ @Override
+ public void applyUpdate() throws FlowUpdateException {
+ final ConnectorState initialDesiredState = connectorNode.getDesiredState();
+
+ connectorNode.prepareForUpdate();
+ connectorNode.applyUpdate();
+
+ if (initialDesiredState == ConnectorState.RUNNING) {
+ connectorNode.start(flowEngine);
+ } else {
+ connectorNode.stop(flowEngine);
+ }
+ }
+
+ @Override
+ public void configure(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) throws FlowUpdateException {
+ connectorNode.setConfiguration(stepName, groupConfigurations);
+ }
+
+ @Override
+ public ConnectorConfigVerificationResult verifyConfiguration(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) {
+ final List<ConfigVerificationResult> results = connectorNode.verifyConfigurationStep(stepName, groupConfigurations);
+ return new MockServerConfigVerificationResult(results);
+ }
+
+ @Override
+ public void startConnector() {
+ initialFlowFileTransferCounts = connectorNode.getFlowFileTransferCounts();
+
+ connectorNode.start(flowEngine);
+ }
+
+ @Override
+ public void stopConnector() {
+ try {
+ connectorNode.stop(flowEngine).get(10, TimeUnit.SECONDS);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for connector to stop", e);
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to stop Connector", e);
+ }
+ }
+
+ @Override
+ public void waitForDataIngested(final Duration maxWaitTime) {
+ final long startTime = System.currentTimeMillis();
+ final long expirationTime = startTime + maxWaitTime.toMillis();
+
+ while (getTransferCountsSinceStart().getReceivedCount() == 0L) {
+ if (System.currentTimeMillis() > expirationTime) {
+ throw new RuntimeException("Timed out waiting for data to be ingested by the Connector");
+ }
+
+ try {
+ Thread.sleep(100L);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for data to be ingested by the Connector", e);
+ }
+ }
+ }
+
+ private FlowFileTransferCounts getTransferCountsSinceStart() {
+ final FlowFileTransferCounts currentCounts = connectorNode.getFlowFileTransferCounts();
+ return currentCounts.minus(initialFlowFileTransferCounts);
+ }
+
+ @Override
+ public void waitForIdle(final Duration maxWaitTime) {
+ waitForIdle(Duration.ofMillis(0L), maxWaitTime);
+ }
+
+ @Override
+ public void waitForIdle(final Duration minimumIdleTime, final Duration maxWaitTime) {
+ Optional<Duration> idleTime = connectorNode.getIdleDuration();
+
+ // Wait until idleTime is not empty and is at least equal to minimumIdleTime
+ final long startTime = System.currentTimeMillis();
+ final long expirationTime = startTime + maxWaitTime.toMillis();
+
+ while (idleTime.isEmpty() || idleTime.get().compareTo(minimumIdleTime) <= 0) {
+ if (System.currentTimeMillis() > expirationTime) {
+ throw new RuntimeException("Timed out waiting for Connector to be idle");
+ }
+
+ try {
+ Thread.sleep(100L);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for Connector to be idle", e);
+ }
+
+ idleTime = connectorNode.getIdleDuration();
+ }
+ }
+
+ @Override
+ public List<ValidationResult> validate() {
+ final ValidationState validationState = connectorNode.performValidation();
+ return validationState.getValidationErrors().stream()
+ .filter(result -> !result.isValid())
+ .filter(result -> !DisabledServiceValidationResult.isMatch(result))
+ .toList();
+ }
+
+ @Override
+ public void mockProcessor(final String processorType, final Class<? extends Processor> mockProcessorClass) {
+ mockExtensionMapper.mockProcessor(processorType, mockProcessorClass.getName());
+ extensionManager.addProcessor(mockProcessorClass);
+ }
+
+ @Override
+ public void close() {
+ stop();
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.NiFiServer
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.NiFiServer
index 56740ff7c3..70572a99ea 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.NiFiServer
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.mock.connector.server.StandardConnectorMockServer
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.components.connector.ConnectorRepository
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.components.connector.ConnectorRepository
index 56740ff7c3..d9230afaec 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/resources/META-INF/services/org.apache.nifi.components.connector.ConnectorRepository
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.mock.connector.server.MockConnectorRepository
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-server/src/test/java/org/apache/nifi/mock/connector/server/MockStateManagerProviderTest.java b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/test/java/org/apache/nifi/mock/connector/server/MockStateManagerProviderTest.java
new file mode 100644
index 0000000000..d8ecf8e872
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-server/src/test/java/org/apache/nifi/mock/connector/server/MockStateManagerProviderTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector.server;
+
+import org.apache.nifi.components.state.Scope;
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.components.state.StateMap;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class MockStateManagerProviderTest {
+
+ @Test
+ void testGetStateManagerReturnsSameInstance() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ final StateManager stateManager2 = provider.getStateManager("component1", false);
+
+ assertNotNull(stateManager1);
+ assertSame(stateManager1, stateManager2);
+ }
+
+ @Test
+ void testGetStateManagerReturnsDifferentInstancesForDifferentComponents() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ final StateManager stateManager2 = provider.getStateManager("component2", false);
+
+ assertNotNull(stateManager1);
+ assertNotNull(stateManager2);
+ assertNotSame(stateManager1, stateManager2);
+ }
+
+ @Test
+ void testOnComponentRemoved() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ assertNotNull(stateManager1);
+
+ provider.onComponentRemoved("component1");
+
+ final StateManager stateManager2 = provider.getStateManager("component1", false);
+ assertNotNull(stateManager2);
+ assertNotSame(stateManager1, stateManager2);
+ }
+
+ @Test
+ void testShutdown() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ provider.getStateManager("component1", false);
+ provider.getStateManager("component2", false);
+
+ provider.shutdown();
+
+ final StateManager stateManager = provider.getStateManager("component1", false);
+ assertNotNull(stateManager);
+ }
+
+ @Test
+ void testClusterProviderEnabled() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ assertFalse(provider.isClusterProviderEnabled());
+
+ provider.enableClusterProvider();
+ assertTrue(provider.isClusterProviderEnabled());
+
+ provider.disableClusterProvider();
+ assertFalse(provider.isClusterProviderEnabled());
+ }
+
+ @Test
+ void testStateManagerSetAndGetLocalState() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state = new HashMap<>();
+ state.put("key1", "value1");
+ state.put("key2", "value2");
+
+ stateManager.setState(state, Scope.LOCAL);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ assertNotNull(stateMap);
+ assertTrue(stateMap.getStateVersion().isPresent());
+ assertEquals("value1", stateMap.get("key1"));
+ assertEquals("value2", stateMap.get("key2"));
+ assertEquals(2, stateMap.toMap().size());
+ }
+
+ @Test
+ void testStateManagerSetAndGetClusterState() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state = new HashMap<>();
+ state.put("key1", "value1");
+ state.put("key2", "value2");
+
+ stateManager.setState(state, Scope.CLUSTER);
+
+ final StateMap stateMap = stateManager.getState(Scope.CLUSTER);
+ assertNotNull(stateMap);
+ assertTrue(stateMap.getStateVersion().isPresent());
+ assertEquals("value1", stateMap.get("key1"));
+ assertEquals("value2", stateMap.get("key2"));
+ }
+
+ @Test
+ void testStateManagerReplace() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> initialState = new HashMap<>();
+ initialState.put("key1", "value1");
+ stateManager.setState(initialState, Scope.LOCAL);
+
+ final StateMap oldStateMap = stateManager.getState(Scope.LOCAL);
+
+ final Map<String, String> newState = new HashMap<>();
+ newState.put("key1", "value2");
+
+ final boolean replaced = stateManager.replace(oldStateMap, newState, Scope.LOCAL);
+ assertTrue(replaced);
+
+ final StateMap updatedStateMap = stateManager.getState(Scope.LOCAL);
+ assertEquals("value2", updatedStateMap.get("key1"));
+
+ final boolean replacedAgain = stateManager.replace(oldStateMap, initialState, Scope.LOCAL);
+ assertFalse(replacedAgain);
+ }
+
+ @Test
+ void testStateManagerClear() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state = new HashMap<>();
+ state.put("key1", "value1");
+ stateManager.setState(state, Scope.LOCAL);
+
+ stateManager.clear(Scope.LOCAL);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ assertNotNull(stateMap);
+ assertTrue(stateMap.toMap().isEmpty());
+ }
+
+ @Test
+ void testStateKeyDropSupported() {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager1 = provider.getStateManager("component1", false);
+ assertFalse(stateManager1.isStateKeyDropSupported());
+
+ final StateManager stateManager2 = provider.getStateManager("component2", true);
+ assertTrue(stateManager2.isStateKeyDropSupported());
+ }
+
+ @Test
+ void testThreadSafety() throws InterruptedException, IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+ final int threadCount = 10;
+ final int operationsPerThread = 100;
+ final ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch completionLatch = new CountDownLatch(threadCount);
+ final AtomicInteger successCount = new AtomicInteger(0);
+
+ for (int i = 0; i < threadCount; i++) {
+ final int threadId = i;
+ executorService.submit(() -> {
+ try {
+ startLatch.await();
+ for (int j = 0; j < operationsPerThread; j++) {
+ final Map<String, String> state = new HashMap<>();
+ state.put("thread", String.valueOf(threadId));
+ state.put("operation", String.valueOf(j));
+ stateManager.setState(state, Scope.LOCAL);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ if (stateMap != null && stateMap.toMap().size() == 2) {
+ successCount.incrementAndGet();
+ }
+ }
+ } catch (final Exception e) {
+ e.printStackTrace();
+ } finally {
+ completionLatch.countDown();
+ }
+ });
+ }
+
+ startLatch.countDown();
+ completionLatch.await(30, TimeUnit.SECONDS);
+ executorService.shutdown();
+
+ assertEquals(threadCount * operationsPerThread, successCount.get());
+ final StateMap finalState = stateManager.getState(Scope.LOCAL);
+ assertNotNull(finalState);
+ assertEquals(2, finalState.toMap().size());
+ }
+
+ @Test
+ void testEmptyStateMap() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final StateMap stateMap = stateManager.getState(Scope.LOCAL);
+ assertNotNull(stateMap);
+ assertFalse(stateMap.getStateVersion().isPresent());
+ assertTrue(stateMap.toMap().isEmpty());
+ }
+
+ @Test
+ void testVersionIncrementing() throws IOException {
+ final MockStateManagerProvider provider = new MockStateManagerProvider();
+ final StateManager stateManager = provider.getStateManager("component1", false);
+
+ final Map<String, String> state1 = new HashMap<>();
+ state1.put("key", "value1");
+ stateManager.setState(state1, Scope.LOCAL);
+ final StateMap stateMap1 = stateManager.getState(Scope.LOCAL);
+
+ final Map<String, String> state2 = new HashMap<>();
+ state2.put("key", "value2");
+ stateManager.setState(state2, Scope.LOCAL);
+ final StateMap stateMap2 = stateManager.getState(Scope.LOCAL);
+
+ assertTrue(stateMap1.getStateVersion().isPresent());
+ assertTrue(stateMap2.getStateVersion().isPresent());
+
+ final long version1 = Long.parseLong(stateMap1.getStateVersion().get());
+ final long version2 = Long.parseLong(stateMap2.getStateVersion().get());
+ assertTrue(version2 > version1);
+ }
+}
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/pom.xml
new file mode 100644
index 0000000000..959ac55052
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/pom.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-integration-tests</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- NARs required for integration testing -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-server-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-connectors-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+
+ <!-- NARs for components referenced in the Connectors -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-shared-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-update-attribute-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-nar</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/libDir</outputDirectory>
+ <includeTypes>jar,nar</includeTypes>
+ <includeScope>compile</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
+
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/CreateConnectorIT.java
similarity index 50%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
copy to nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/CreateConnectorIT.java
index 96806fca1d..dccbaaa705 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/CreateConnectorIT.java
@@ -14,23 +14,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.controller;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
-import org.apache.nifi.parameter.ParameterContext;
-import org.apache.nifi.validation.RuleViolationsManager;
+package org.apache.nifi.mock.connectors.tests;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.nifi.mock.connector.StandardConnectorTestRunner;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.junit.jupiter.api.Test;
-public interface ValidationContextFactory {
+import java.io.File;
+import java.io.IOException;
- ValidationContext newValidationContext(Map<PropertyDescriptor, PropertyConfiguration> properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext,
- boolean validateConnections);
+public class CreateConnectorIT {
- Optional<RuleViolationsManager> getRuleViolationsManager();
+ @Test
+ public void testCreateStartAndStopGenerateAndUpdateConnector() throws IOException {
+ try (final ConnectorTestRunner testRunner = new StandardConnectorTestRunner.Builder()
+ .connectorClassName("org.apache.nifi.mock.connectors.GenerateAndLog")
+ .narLibraryDirectory(new File("target/libDir"))
+ .build()) {
- Optional<FlowAnalyzer> getFlowAnalyzer();
+ testRunner.startConnector();
+ testRunner.stopConnector();
+ }
+ }
}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/MockProcessorIT.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/MockProcessorIT.java
new file mode 100644
index 0000000000..465d034000
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-integration-tests/src/test/java/org/apache/nifi/mock/connectors/tests/MockProcessorIT.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connectors.tests;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.mock.connector.StandardConnectorTestRunner;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MockProcessorIT {
+
+ @Test
+ @Timeout(10)
+ public void testMockProcessor() throws IOException {
+ try (final ConnectorTestRunner runner = new StandardConnectorTestRunner.Builder()
+ .narLibraryDirectory(new File("target/libDir"))
+ .connectorClassName("org.apache.nifi.mock.connectors.GenerateAndLog")
+ .mockProcessor("org.apache.nifi.processors.attributes.UpdateAttribute", MockProcessor.class)
+ .build()) {
+
+ runner.startConnector();
+
+ // Wait until MockProcessor is triggered at least once. We use @Timeout on the test to avoid
+ // hanging indefinitely in case of failure.
+ while (MockProcessor.invocationCounter.get() < 1) {
+ Thread.yield();
+ }
+ runner.stopConnector();
+ }
+ }
+
+ public static class MockProcessor extends AbstractProcessor {
+ private static final AtomicLong invocationCounter = new AtomicLong(0L);
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .build();
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return Set.of(REL_SUCCESS);
+ }
+
+ // Support any properties so that properties of UpdateAttribute can be set without the Processor becoming invalid
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .name(propertyDescriptorName)
+ .addValidator(Validator.VALID)
+ .build();
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ invocationCounter.incrementAndGet();
+
+ final FlowFile flowFile = session.get();
+ if (flowFile != null) {
+ session.transfer(flowFile, REL_SUCCESS);
+ }
+ }
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors-nar/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors-nar/pom.xml
new file mode 100644
index 0000000000..89d3a10121
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors-nar/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-test-connectors-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-connectors</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/pom.xml
new file mode 100644
index 0000000000..9f2a86cd96
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-test-connectors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java
new file mode 100644
index 0000000000..f9775bc2d8
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/java/org/apache/nifi/mock/connectors/GenerateAndLog.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connectors;
+
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.connector.AbstractConnector;
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.components.connector.components.FlowContext;
+import org.apache.nifi.components.connector.util.VersionedFlowUtils;
+import org.apache.nifi.flow.VersionedExternalFlow;
+
+import java.util.List;
+import java.util.Map;
+
+public class GenerateAndLog extends AbstractConnector {
+
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return VersionedFlowUtils.loadFlowFromResource("flows/Generate_and_Update.json");
+ }
+
+ @Override
+ protected void onStepConfigured(final String stepName, final FlowContext flowContext) {
+ }
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps(final FlowContext flowContext) {
+ return List.of();
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException {
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final String stepName, final List<PropertyGroupConfiguration> overrides, final FlowContext workingContext) {
+ return List.of();
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
index 56740ff7c3..c6cbc30195 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.mock.connectors.GenerateAndLog
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Generate_and_Update.json b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Generate_and_Update.json
new file mode 100644
index 0000000000..686247011e
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Generate_and_Update.json
@@ -0,0 +1 @@
+{"flowContents":{"identifier":"cdf7f935-0045-3c88-bdad-fd4e34105756","instanceIdentifier":"3a87b58f-019a-1000-30d2-87dfdd9816aa","name":"Generate and Update","comments":"","position":{"x":-1124.0,"y":65.5},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"4d4160b1-736c-3be4-bc3a-84fc6eb4ad2a","instanceIdentifier":"3a87db5f-019a-1000-75df-90e055f21fc9","name":"UpdateAttribute","comments":"","position":{"x":-358.0,"y":-106.5},"type":"org.apache.nifi.processors.attrib [...]
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/pom.xml
new file mode 100644
index 0000000000..c7b734eb41
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/pom.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-test-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-connector-mock-test-connectors</module>
+ <module>nifi-connector-mock-test-connectors-nar</module>
+ <module>nifi-connector-mock-integration-tests</module>
+ </modules>
+</project>
+
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/pom.xml b/nifi-connector-mock-bundle/nifi-connector-mock/pom.xml
new file mode 100644
index 0000000000..3f6ea8b8cd
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock/pom.xml
@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-properties</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Dependencies necessary to bootstrap NiFi -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+
+ <!-- Core NiFi API dependencies -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-python-framework-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-stateless-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Logging dependencies to ensure that everything funnels through SLF4J -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- NARs required -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-jetty-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-provenance-repository-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-nar</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/libDir</outputDirectory>
+ <includeTypes>jar,nar</includeTypes>
+ <includeScope>compile</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/StandardConnectorTestRunner.java b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/StandardConnectorTestRunner.java
new file mode 100644
index 0000000000..907204d0f0
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/StandardConnectorTestRunner.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.mock.connector;
+
+import org.apache.nifi.NiFiServer;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.mock.connector.server.ConnectorConfigVerificationResult;
+import org.apache.nifi.mock.connector.server.ConnectorMockServer;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.apache.nifi.nar.ExtensionMapping;
+import org.apache.nifi.nar.NarClassLoaders;
+import org.apache.nifi.nar.NarUnpackMode;
+import org.apache.nifi.nar.NarUnpacker;
+import org.apache.nifi.nar.SystemBundle;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.util.NiFiProperties;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class StandardConnectorTestRunner implements ConnectorTestRunner, Closeable {
+ private final File narLibraryDirectory;
+
+ private ConnectorMockServer mockServer;
+
+ private StandardConnectorTestRunner(final Builder builder) {
+ this.narLibraryDirectory = builder.narLibraryDirectory;
+
+ try {
+ bootstrapInstance();
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to bootstrap ConnectorTestRunner", e);
+ }
+
+ // It is important that we register the processor mocks before instantiating the connector.
+ // Otherwise, the call to instantiateConnector will initialize the Connector, which may update the flow.
+ // If the flow is updated before the processor mocks are registered, the Processors will be created without
+ // using the mocks. Subsequent updates to the flow will not replace the Processors already created because
+ // these are not recognized as updates to the flow, since the framework assumes that the type of a Processor
+ // with a given ID does not change.
+ builder.processorMocks.forEach(mockServer::mockProcessor);
+
+ mockServer.instantiateConnector(builder.connectorClassName);
+ }
+
+ private void bootstrapInstance() throws IOException, ClassNotFoundException {
+ final List<Path> libDirectoryPaths = List.of(narLibraryDirectory.toPath());
+ final File extensionsWorkingDir = new File("target/work/extensions");
+ final File frameworkWorkingDir = new File("target/work/framework");
+
+ final Bundle systemBundle = SystemBundle.create(narLibraryDirectory.getAbsolutePath(), ClassLoader.getSystemClassLoader());
+
+ final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(systemBundle, frameworkWorkingDir, extensionsWorkingDir, libDirectoryPaths, true,
+ NarClassLoaders.FRAMEWORK_NAR_ID, true, false, NarUnpackMode.UNPACK_INDIVIDUAL_JARS, bundleCoordinate -> true);
+
+ final NarClassLoaders narClassLoaders = new NarClassLoaders();
+ narClassLoaders.init(frameworkWorkingDir, extensionsWorkingDir);
+
+ final NiFiServer nifiServer = narClassLoaders.getServer();
+ if (nifiServer == null) {
+ throw new RuntimeException("Could not find NiFiServer instance");
+ }
+ if (!(nifiServer instanceof ConnectorMockServer)) {
+ throw new RuntimeException("Test ClassPath does not contain ConnectorMockServer. " +
+ "Ensure that the appropriate module is packaged in the NAR library directory: " + narLibraryDirectory.getAbsolutePath());
+ }
+
+ // Set Application Server Class Loader for subsequent operations
+ final ClassLoader applicationServerClassLoader = narClassLoaders.getServer().getClass().getClassLoader();
+ Thread.currentThread().setContextClassLoader(applicationServerClassLoader);
+
+ final Set<Bundle> narBundles = narClassLoaders.getBundles();
+
+ final NiFiProperties properties;
+ try (final InputStream propertiesIn = getClass().getClassLoader().getResourceAsStream("nifi.properties")) {
+ properties = NiFiProperties.createBasicNiFiProperties(propertiesIn);
+ }
+
+ nifiServer.initialize(properties, systemBundle, narBundles, extensionMapping);
+ nifiServer.start();
+
+ mockServer = (ConnectorMockServer) nifiServer;
+ mockServer.registerMockBundle(getClass().getClassLoader(), new File(extensionsWorkingDir, "mock-implementations-bundle"));
+ }
+
+ @Override
+ public void close() {
+ if (mockServer != null) {
+ mockServer.stop();
+ }
+ }
+
+ @Override
+ public void applyUpdate() throws FlowUpdateException {
+ mockServer.applyUpdate();
+ }
+
+ @Override
+ public void configure(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) throws FlowUpdateException {
+ mockServer.configure(stepName, groupConfigurations);
+ }
+
+ @Override
+ public ConnectorConfigVerificationResult verifyConfiguration(final String stepName, final List<PropertyGroupConfiguration> groupConfigurations) {
+ return mockServer.verifyConfiguration(stepName, groupConfigurations);
+ }
+
+ @Override
+ public void startConnector() {
+ mockServer.startConnector();
+ }
+
+ @Override
+ public void stopConnector() {
+ mockServer.stopConnector();
+ }
+
+ @Override
+ public void waitForDataIngested(final Duration maxWaitTime) {
+ mockServer.waitForDataIngested(maxWaitTime);
+ }
+
+ @Override
+ public void waitForIdle(final Duration maxWaitTime) {
+ mockServer.waitForIdle(maxWaitTime);
+ }
+
+ @Override
+ public void waitForIdle(final Duration minIdleTime, final Duration maxWaitTime) {
+ mockServer.waitForIdle(minIdleTime, maxWaitTime);
+ }
+
+ @Override
+ public List<ValidationResult> validate() {
+ return mockServer.validate();
+ }
+
+
+ public static class Builder {
+ private String connectorClassName;
+ private File narLibraryDirectory;
+ private final Map<String, Class<? extends Processor>> processorMocks = new HashMap<>();
+
+ public Builder connectorClassName(final String connectorClassName) {
+ this.connectorClassName = connectorClassName;
+ return this;
+ }
+
+ public Builder narLibraryDirectory(final File libDirectory) {
+ this.narLibraryDirectory = libDirectory;
+ return this;
+ }
+
+ public Builder mockProcessor(final String processorType, final Class<? extends Processor> mockProcessorClass) {
+ processorMocks.put(processorType, mockProcessorClass);
+ return this;
+ }
+
+ public StandardConnectorTestRunner build() {
+ if (!narLibraryDirectory.exists() || !narLibraryDirectory.isDirectory()) {
+ throw new IllegalArgumentException("NAR file does not exist or is not a directory: " + narLibraryDirectory.getAbsolutePath());
+ }
+
+ return new StandardConnectorTestRunner(this);
+ }
+ }
+}
diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties
new file mode 100644
index 0000000000..046a2980b2
--- /dev/null
+++ b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties
@@ -0,0 +1,159 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Core Properties #
+nifi.flow.configuration.file=target/nifi-storage/conf/flow.json.gz
+nifi.flow.configuration.archive.enabled=true
+nifi.flow.configuration.archive.dir=target/nifi-storage/conf/archive/
+nifi.flow.configuration.archive.max.time=30 days
+nifi.flow.configuration.archive.max.storage=500 MB
+nifi.flow.configuration.archive.max.count=
+nifi.flowcontroller.autoResumeState=true
+nifi.flowcontroller.graceful.shutdown.period=10 sec
+nifi.flowservice.writedelay.interval=500 ms
+nifi.administrative.yield.duration=3 sec
+# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
+nifi.bored.yield.duration=10 millis
+nifi.queue.backpressure.count=10000
+nifi.queue.backpressure.size=1 GB
+
+nifi.nar.library.directory=./lib
+nifi.nar.library.autoload.directory=./extensions
+nifi.nar.working.directory=./work/nar/
+
+#####################
+# Python Extensions #
+#####################
+# Uncomment in order to enable Python Extensions.
+#nifi.python.command=python
+nifi.python.framework.source.directory=./python/framework
+nifi.python.extensions.source.directory.default=./python/extensions
+nifi.python.working.directory=./work/python
+nifi.python.max.processes=100
+nifi.python.max.processes.per.extension.type=10
+
+####################
+# State Management #
+####################
+nifi.state.management.configuration.file=target/nifi-storage/state-management.xml
+# The ID of the local state provider
+nifi.state.management.provider.local=local-provider
+
+# Database Settings
+nifi.database.directory=target/nifi-storage/database_repository
+
+# FlowFile Repository
+nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
+nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog
+nifi.flowfile.repository.directory=target/nifi-storage/flowfile_repository
+nifi.flowfile.repository.checkpoint.interval=1 mins
+nifi.flowfile.repository.always.sync=false
+
+nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
+nifi.queue.swap.threshold=20000
+nifi.swap.in.period=5 sec
+nifi.swap.in.threads=1
+nifi.swap.out.period=5 sec
+nifi.swap.out.threads=4
+
+# Content Repository
+nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
+nifi.content.claim.max.appendable.size=50 KB
+nifi.content.repository.directory.default=target/nifi-storage/content_repository
+nifi.content.repository.archive.max.retention.period=12 hours
+nifi.content.repository.archive.max.usage.percentage=90%
+nifi.content.repository.archive.enabled=false
+nifi.content.repository.always.sync=false
+
+# Provenance Repository Properties
+nifi.provenance.repository.implementation=org.apache.nifi.provenance.VolatileProvenanceRepository
+
+# Volatile Provenance Respository Properties
+nifi.provenance.repository.buffer.size=1000
+
+# Component Status Repository
+nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
+nifi.components.status.repository.buffer.size=1440
+nifi.components.status.snapshot.frequency=1 min
+
+# NAR Persistence Provider Properties
+nifi.nar.persistence.provider.properties.directory=target/nifi-storage/nar_repository
+
+# Asset Management
+nifi.asset.manager.properties.directory=target/nifi-storage/assets
+
+# Site to Site properties
+nifi.remote.input.host=localhost
+nifi.remote.input.secure=true
+nifi.remote.input.socket.port=7780
+nifi.remote.input.http.enabled=true
+nifi.remote.input.http.transaction.ttl=30 sec
+nifi.remote.contents.cache.expiration=30 secs
+
+# web properties #
+nifi.web.war.directory=./lib
+nifi.web.http.host=
+nifi.web.http.port=
+nifi.web.http.network.interface.default=
+nifi.web.https.host=localhost
+nifi.web.https.port=0
+nifi.web.https.network.interface.default=
+nifi.web.jetty.working.directory=target/work/jetty
+nifi.web.jetty.threads=200
+nifi.web.max.header.size=16 KB
+nifi.web.proxy.context.path=
+nifi.web.proxy.host=
+
+# security properties #
+nifi.sensitive.props.key=nifi-connector-mock-tests
+nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
+
+nifi.security.keystore=
+nifi.security.keystoreType=PKCS12
+nifi.security.keystorePasswd=
+nifi.security.keyPasswd=
+nifi.security.truststore=
+nifi.security.truststoreType=PKCS12
+nifi.security.truststorePasswd=
+nifi.security.user.authorizer=system-test-authorizer
+nifi.security.user.login.identity.provider=
+
+# cluster common properties (all nodes must have same values) #
+nifi.cluster.protocol.heartbeat.interval=5 sec
+nifi.cluster.protocol.is.secure=true
+
+# cluster node properties (only configure for cluster nodes) #
+nifi.cluster.is.node=false
+nifi.cluster.node.address=
+nifi.cluster.node.protocol.port=
+nifi.cluster.node.protocol.threads=10
+nifi.cluster.node.protocol.max.threads=50
+nifi.cluster.node.event.history.size=25
+nifi.cluster.node.connection.timeout=5 sec
+nifi.cluster.node.read.timeout=5 sec
+nifi.cluster.node.max.concurrent.requests=100
+nifi.cluster.firewall.file=
+nifi.cluster.flow.election.max.wait.time=5 mins
+nifi.cluster.flow.election.max.candidates=
+
+# cluster load balancing properties #
+nifi.cluster.load.balance.host=
+nifi.cluster.load.balance.port=6342
+nifi.cluster.load.balance.connections.per.node=4
+nifi.cluster.load.balance.max.thread.count=8
+nifi.cluster.load.balance.comms.timeout=30 sec
+
+# Connector Repository
+nifi.components.connectors.repository.implementation=org.apache.nifi.mock.connector.server.MockConnectorRepository
diff --git a/nifi-connector-mock-bundle/pom.xml b/nifi-connector-mock-bundle/pom.xml
new file mode 100644
index 0000000000..adb016336e
--- /dev/null
+++ b/nifi-connector-mock-bundle/pom.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connector-mock-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-connector-mock</module>
+ <module>nifi-connector-mock-api</module>
+ <module>nifi-connector-mock-server</module>
+ <module>nifi-connector-mock-server-nar</module>
+ <module>nifi-connector-mock-test-bundle</module>
+ </modules>
+</project>
\ No newline at end of file
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/pom.xml
new file mode 100644
index 0000000000..7f95903475
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-connector</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-utils</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
new file mode 100644
index 0000000000..793b1e9be0
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaConnectionStep.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorPropertyDescriptor;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.PropertyType;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.List;
+
+public class KafkaConnectionStep {
+ public static final String STEP_NAME = "Kafka Connection";
+
+ public static final ConnectorPropertyDescriptor KAFKA_BROKERS = new ConnectorPropertyDescriptor.Builder()
+ .name("Kafka Brokers")
+ .description("A comma-separated list of Kafka brokers to connect to.")
+ .required(true)
+ .validators(StandardValidators.HOSTNAME_PORT_LIST_VALIDATOR)
+ .type(PropertyType.STRING_LIST)
+ .build();
+
+ public static final ConnectorPropertyDescriptor SECURITY_PROTOCOL = new ConnectorPropertyDescriptor.Builder()
+ .name("Security Protocol")
+ .description("The security protocol to use when connecting to Kafka brokers.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .defaultValue("SASL_PLAINTEXT")
+ .allowableValues("PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL")
+ .build();
+
+ public static final ConnectorPropertyDescriptor SASL_MECHANISM = new ConnectorPropertyDescriptor.Builder()
+ .name("SASL Mechanism")
+ .description("The SASL mechanism to use for authentication.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .allowableValues("PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "GSSAPI", "OAUTHBEARER")
+ .dependsOn(SECURITY_PROTOCOL, "SASL_PLAINTEXT", "SASL_SSL")
+ .defaultValue("SCRAM-SHA-512")
+ .build();
+
+ public static final ConnectorPropertyDescriptor USERNAME = new ConnectorPropertyDescriptor.Builder()
+ .name("Username")
+ .description("The username for SASL authentication.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SECURITY_PROTOCOL, "SASL_PLAINTEXT", "SASL_SSL")
+ .build();
+
+ public static final ConnectorPropertyDescriptor PASSWORD = new ConnectorPropertyDescriptor.Builder()
+ .name("Password")
+ .description("The password for SASL authentication.")
+ .required(true)
+ .type(PropertyType.SECRET)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SECURITY_PROTOCOL, "SASL_PLAINTEXT", "SASL_SSL")
+ .build();
+
+ public static final ConnectorPropertyGroup KAFKA_SERVER_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Kafka Server Settings")
+ .description("Settings for connecting to the Kafka server")
+ .properties(List.of(
+ KAFKA_BROKERS,
+ SECURITY_PROTOCOL,
+ SASL_MECHANISM,
+ USERNAME,
+ PASSWORD
+ ))
+ .build();
+
+ public static final ConnectorPropertyDescriptor SCHEMA_REGISTRY_URL = new ConnectorPropertyDescriptor.Builder()
+ .name("Schema Registry URL")
+ .description("The URL of the Schema Registry.")
+ .required(false)
+ .type(PropertyType.STRING)
+ .validators(StandardValidators.URL_VALIDATOR)
+ .validators(Validator.VALID)
+ .build();
+
+ public static final ConnectorPropertyDescriptor SCHEMA_REGISTRY_USERNAME = new ConnectorPropertyDescriptor.Builder()
+ .name("Schema Registry Username")
+ .description("The username for Schema Registry authentication.")
+ .required(false)
+ .type(PropertyType.STRING)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SCHEMA_REGISTRY_URL)
+ .build();
+
+ public static final ConnectorPropertyDescriptor SCHEMA_REGISTRY_PASSWORD = new ConnectorPropertyDescriptor.Builder()
+ .name("Schema Registry Password")
+ .description("The password for Schema Registry authentication.")
+ .required(false)
+ .type(PropertyType.SECRET)
+ .validators(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(SCHEMA_REGISTRY_URL)
+ .build();
+
+
+ public static final ConnectorPropertyGroup SCHEMA_REGISTRY_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Schema Registry Settings")
+ .description("Settings for connecting to the Schema Registry")
+ .properties(List.of(
+ SCHEMA_REGISTRY_URL,
+ SCHEMA_REGISTRY_USERNAME,
+ SCHEMA_REGISTRY_PASSWORD
+ ))
+ .build();
+
+ public static final ConfigurationStep KAFKA_CONNECTION_STEP = new ConfigurationStep.Builder()
+ .name(STEP_NAME)
+ .description("Configure Kafka connection settings")
+ .propertyGroups(List.of(
+ KAFKA_SERVER_GROUP,
+ SCHEMA_REGISTRY_GROUP
+ ))
+ .build();
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java
new file mode 100644
index 0000000000..c94226e132
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
+import org.apache.nifi.components.connector.AbstractConnector;
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorConfigurationContext;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.InvocationFailedException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.components.connector.components.ControllerServiceFacade;
+import org.apache.nifi.components.connector.components.FlowContext;
+import org.apache.nifi.components.connector.components.ProcessGroupFacade;
+import org.apache.nifi.components.connector.components.ProcessorFacade;
+import org.apache.nifi.components.connector.util.VersionedFlowUtils;
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.flow.VersionedExternalFlow;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+@CapabilityDescription("Provides the ability to ingest data from Apache Kafka topics, merge it together into an object of reasonable " +
+ "size, and write that data to Amazon S3.")
+@Tags({"kafka", "s3"})
+public class KafkaToS3 extends AbstractConnector {
+
+ private volatile CompletableFuture<Void> drainFlowFileFuture = null;
+
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps(final FlowContext flowContext) {
+ return List.of(
+ KafkaConnectionStep.KAFKA_CONNECTION_STEP,
+ KafkaTopicsStep.KAFKA_TOPICS_STEP,
+ S3Step.S3_STEP
+ );
+ }
+
+ @Override
+ public void prepareForUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException {
+ final String activeS3DataFormat = activeContext.getConfigurationContext().getProperty(
+ S3Step.S3_STEP_NAME, S3Step.S3_DESTINATION_GROUP.getName(), S3Step.S3_DATA_FORMAT.getName()).getValue();
+ final String workingS3DataFormat = workingContext.getConfigurationContext().getProperty(
+ S3Step.S3_STEP_NAME, S3Step.S3_DESTINATION_GROUP.getName(), S3Step.S3_DATA_FORMAT.getName()).getValue();
+
+ if (!activeS3DataFormat.equals(workingS3DataFormat)) {
+ getLogger().info("S3 Data Format changed from {} to {}; draining flow before updating it", activeS3DataFormat, workingS3DataFormat);
+
+ drainFlowFileFuture = drainFlowFiles(activeContext);
+ try {
+ drainFlowFileFuture.get(5, TimeUnit.MINUTES);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new FlowUpdateException("Interrupted while waiting for all FlowFiles to drain from the flow", e);
+ } catch (final TimeoutException e) {
+ throw new FlowUpdateException("Timed out waiting for all FlowFiles to drain from the flow", e);
+ } catch (final ExecutionException e) {
+ throw new FlowUpdateException("Failed to drain FlowFiles from flow", e.getCause());
+ }
+
+ getLogger().info("All FlowFiles drained from the flow; proceeding with flow update");
+ }
+ }
+
+ @Override
+ public void abortUpdate(final FlowContext workingContext, final Throwable throwable) {
+ if (drainFlowFileFuture != null) {
+ drainFlowFileFuture.cancel(true);
+ drainFlowFileFuture = null;
+ }
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final FlowContext activeContext) throws FlowUpdateException {
+ final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext());
+ getInitializationContext().updateFlow(activeContext, flow);
+ }
+
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return KafkaToS3FlowBuilder.loadInitialFlow();
+ }
+
+ @Override
+ public void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException {
+ final VersionedExternalFlow flow = buildFlow(workingContext.getConfigurationContext());
+ getInitializationContext().updateFlow(workingContext, flow);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final String stepName, final List<PropertyGroupConfiguration> propertyOverrides, final FlowContext workingFlowContext) {
+ // Get the current ConfigurationContext and then create a new one that contains the provided property values
+ final ConnectorConfigurationContext configurationContext = workingFlowContext.getConfigurationContext().createWithOverrides(stepName, propertyOverrides);
+ final VersionedExternalFlow flow = buildFlow(configurationContext);
+
+ // Validate Connectivity
+ if (stepName.equals(KafkaConnectionStep.STEP_NAME)) {
+ return verifyKafkaConnectivity(workingFlowContext, flow);
+ }
+ if (stepName.equals(KafkaTopicsStep.STEP_NAME)) {
+ final List<ConfigVerificationResult> results = new ArrayList<>();
+ results.addAll(verifyTopicsExists(workingFlowContext));
+ results.addAll(verifyKafkaParsability(workingFlowContext, flow));
+ return results;
+ }
+
+ return Collections.emptyList();
+ }
+
+ private List<ConfigVerificationResult> verifyKafkaParsability(final FlowContext workingFlowContext, final VersionedExternalFlow flow) {
+ // Enable Controller Services necessary for parsing records.
+ // We determine which Controller Services are referenced by the flow and enable them, but we do not use
+ // getRootGroup().getLifecycle().enableReferencedControllerServices(ControllerServiceReferenceScope.INCLUDE_REFERENCED_SERVICES_ONLY)
+ // because that would include the Controller Services that are referenced by the currently configured flow, and it's possible that the
+ // what is being verified uses a different set of Controller Services (e.g., the verified flow may use a JSON Reader while the current
+ // flow uses an Avro Reader).
+ final ProcessGroupFacade rootGroup = workingFlowContext.getRootGroup();
+ final Set<VersionedControllerService> referencedServices = VersionedFlowUtils.getReferencedControllerServices(flow.getFlowContents());
+ final Set<String> serviceIds = referencedServices.stream()
+ .map(VersionedControllerService::getIdentifier)
+ .collect(Collectors.toSet());
+
+ try {
+ rootGroup.getLifecycle().enableControllerServices(serviceIds).get(10, TimeUnit.SECONDS);
+ } catch (final Exception e) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Record Parsing")
+ .outcome(Outcome.FAILED)
+ .explanation("Failed to enable Controller Services due to " + e)
+ .build());
+ }
+
+ try {
+ final ProcessorFacade consumeKafkaFacade = findProcessors(rootGroup,
+ processor -> processor.getDefinition().getType().endsWith("ConsumeKafka")).getFirst();
+
+ final List<ConfigVerificationResult> configVerificationResults = consumeKafkaFacade.verify(flow, Map.of());
+ for (final ConfigVerificationResult result : configVerificationResults) {
+ if (result.getOutcome() == Outcome.FAILED) {
+ return List.of(result);
+ }
+ }
+
+ return Collections.emptyList();
+ } finally {
+ rootGroup.getLifecycle().disableControllerServices(serviceIds);
+ }
+ }
+
+
+ private List<ConfigVerificationResult> verifyTopicsExists(final FlowContext workingFlowContext) {
+ final List<String> topicsAvailable;
+ try {
+ topicsAvailable = getAvailableTopics(workingFlowContext);
+ } catch (final Exception e) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka topics exist")
+ .outcome(Outcome.SKIPPED)
+ .explanation("Unable to validate that topics exist due to " + e)
+ .build());
+ }
+
+ final Set<String> topicNames = new HashSet<>(topicsAvailable);
+ final List<String> specifiedTopics = workingFlowContext.getConfigurationContext().getProperty(KafkaTopicsStep.STEP_NAME,
+ KafkaTopicsStep.KAFKA_TOPICS_GROUP.getName(), KafkaTopicsStep.TOPIC_NAMES.getName()).asList();
+ final String missingTopics = specifiedTopics.stream()
+ .filter(topic -> !topicNames.contains(topic))
+ .collect(Collectors.joining(", "));
+
+ if (!missingTopics.isEmpty()) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka topics exist")
+ .outcome(Outcome.FAILED)
+ .explanation("The following topics do not exist in the Kafka cluster: " + missingTopics)
+ .build());
+ } else {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka topics exist")
+ .outcome(Outcome.SUCCESSFUL)
+ .explanation("All specified topics exist in the Kafka cluster")
+ .build());
+ }
+ }
+
+ private List<ConfigVerificationResult> verifyKafkaConnectivity(final FlowContext workingFlowContext, final VersionedExternalFlow flow) {
+ // Build a new version of the flow so that we can get the relevant properties of the Kafka Connection Service
+ final ControllerServiceFacade connectionService = getKafkaConnectionService(workingFlowContext);
+ final List<ConfigVerificationResult> configVerificationResults = connectionService.verify(flow, Map.of());
+
+ for (final ConfigVerificationResult result : configVerificationResults) {
+ if (result.getOutcome() == Outcome.FAILED) {
+ return List.of(new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Kafka connectivity")
+ .outcome(Outcome.FAILED)
+ .explanation(result.getExplanation())
+ .build());
+ }
+ }
+
+ return Collections.emptyList();
+ }
+
+ private VersionedExternalFlow buildFlow(final ConnectorConfigurationContext configurationContext) {
+ final KafkaToS3FlowBuilder flowBuilder = new KafkaToS3FlowBuilder(configurationContext);
+ return flowBuilder.buildFlow();
+ }
+
+ @Override
+ public List<AllowableValue> fetchAllowableValues(final String stepName, final String groupName, final String propertyName, final FlowContext flowContext) {
+ if (stepName.equals(KafkaTopicsStep.STEP_NAME) && propertyName.equals(KafkaTopicsStep.TOPIC_NAMES.getName())) {
+ return createAllowableValues(getAvailableTopics(flowContext));
+ } else if (stepName.equals(S3Step.S3_STEP_NAME) && propertyName.equals(S3Step.S3_REGION.getName())) {
+ return createAllowableValues(getPossibleS3Regions(flowContext));
+ }
+
+ return super.fetchAllowableValues(stepName, groupName, propertyName, flowContext);
+ }
+
+ private List<AllowableValue> createAllowableValues(final List<String> values) {
+ return values.stream().map(this::createAllowableValue).collect(Collectors.toList());
+ }
+
+ private AllowableValue createAllowableValue(final String value) {
+ return new AllowableValue(value, value, value);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> getAvailableTopics(final FlowContext flowContext) {
+ // If Kafka Brokers not yet set, return empty list
+ final ConnectorConfigurationContext config = flowContext.getConfigurationContext();
+ if (!config.getProperty(KafkaConnectionStep.KAFKA_CONNECTION_STEP, KafkaConnectionStep.KAFKA_SERVER_GROUP, KafkaConnectionStep.KAFKA_BROKERS).isSet()) {
+ return List.of();
+ }
+
+ final ControllerServiceFacade kafkaConnectionService = getKafkaConnectionService(flowContext);
+
+ try {
+ return (List<String>) kafkaConnectionService.invokeConnectorMethod("listTopicNames", Map.of());
+ } catch (final Exception e) {
+ getLogger().warn("Failed to retrieve available Kafka topics", e);
+ return List.of();
+ }
+ }
+
+ private ControllerServiceFacade getKafkaConnectionService(final FlowContext flowContext) {
+ return flowContext.getRootGroup().getControllerServices().stream()
+ .filter(service -> service.getDefinition().getType().endsWith("Kafka3ConnectionService"))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> getPossibleS3Regions(final FlowContext flowContext) {
+ final ProcessorFacade processorFacade = flowContext.getRootGroup().getProcessors().stream()
+ .filter(proc -> proc.getDefinition().getType().endsWith("PutS3Object"))
+ .findFirst()
+ .orElseThrow();
+
+ try {
+ return (List<String>) processorFacade.invokeConnectorMethod("getAvailableRegions", Map.of());
+ } catch (final InvocationFailedException e) {
+ getLogger().error("Failed to obtain list of available S3 regions", e);
+ return Collections.emptyList();
+ }
+ }
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3FlowBuilder.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3FlowBuilder.java
new file mode 100644
index 0000000000..88b22df5e3
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3FlowBuilder.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorConfigurationContext;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.util.VersionedFlowUtils;
+import org.apache.nifi.flow.VersionedControllerService;
+import org.apache.nifi.flow.VersionedExternalFlow;
+import org.apache.nifi.flow.VersionedProcessGroup;
+import org.apache.nifi.processor.DataUnit;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaToS3FlowBuilder {
+ private static final String FLOW_JSON_PATH = "flows/Kafka_to_S3.json";
+
+ private final ConnectorConfigurationContext configContext;
+
+ public KafkaToS3FlowBuilder(final ConnectorConfigurationContext configurationContext) {
+ this.configContext = configurationContext;
+ }
+
+ public static VersionedExternalFlow loadInitialFlow() {
+ return VersionedFlowUtils.loadFlowFromResource(FLOW_JSON_PATH);
+ }
+
+ public VersionedExternalFlow buildFlow() {
+ final VersionedExternalFlow externalFlow = VersionedFlowUtils.loadFlowFromResource(FLOW_JSON_PATH);
+ configureSchemaRegistry(externalFlow);
+
+ updateKafkaConnectionParameters(externalFlow);
+ updateSchemaRegistryParameters(externalFlow);
+ updateReaderWriter(externalFlow);
+ updateKafkaTopicsParameters(externalFlow);
+ updateS3Config(externalFlow);
+
+ return externalFlow;
+ }
+
+ private void configureSchemaRegistry(final VersionedExternalFlow externalFlow) {
+ final String schemaRegistryUrl = configContext.getProperty(KafkaConnectionStep.KAFKA_CONNECTION_STEP,
+ KafkaConnectionStep.SCHEMA_REGISTRY_GROUP,
+ KafkaConnectionStep.SCHEMA_REGISTRY_URL).getValue();
+
+ if (schemaRegistryUrl == null) {
+ final VersionedProcessGroup processGroup = externalFlow.getFlowContents();
+
+ // Remove any references to the Schema Registry service.
+ final VersionedControllerService schemaRegistryService = processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("ConfluentSchemaRegistry"))
+ .findFirst()
+ .orElseThrow();
+ VersionedFlowUtils.removeControllerServiceReferences(processGroup, schemaRegistryService.getIdentifier());
+
+ final VersionedControllerService schemaReferenceReader = processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("ConfluentEncodedSchemaReferenceReader"))
+ .findFirst()
+ .orElseThrow();
+ VersionedFlowUtils.removeControllerServiceReferences(processGroup, schemaReferenceReader.getIdentifier());
+
+ processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("JsonTreeReader"))
+ .forEach(service -> service.getProperties().put("Schema Access Strategy", "infer-schema"));
+
+ processGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("JsonRecordSetWriter"))
+ .forEach(service -> service.getProperties().put("Schema Write Strategy", "no-schema"));
+ }
+ }
+
+ private void updateSchemaRegistryParameters(final VersionedExternalFlow externalFlow) {
+ final ConfigurationStep connectionStep = KafkaConnectionStep.KAFKA_CONNECTION_STEP;
+ final ConnectorPropertyGroup registryGroup = KafkaConnectionStep.SCHEMA_REGISTRY_GROUP;
+
+ final String schemaRegistryUrl = configContext.getProperty(connectionStep, registryGroup, KafkaConnectionStep.SCHEMA_REGISTRY_URL).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry URLs", schemaRegistryUrl);
+
+ if (schemaRegistryUrl == null) {
+ final Map<String, String> properties = new HashMap<>();
+ properties.put("schema-access-strategy", "infer-schema");
+ properties.put("schema-registry", null);
+ properties.put("schema-reference-reader", null);
+
+ externalFlow.getFlowContents().getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("JsonTreeReader"))
+ .findFirst()
+ .ifPresent(service -> service.setProperties(properties));
+ } else {
+ final String username = configContext.getProperty(connectionStep, registryGroup, KafkaConnectionStep.SCHEMA_REGISTRY_USERNAME).getValue();
+ final String password = configContext.getProperty(connectionStep, registryGroup, KafkaConnectionStep.SCHEMA_REGISTRY_PASSWORD).getValue();
+
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry Username", username);
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry Password", password);
+
+ final String authenticationType = (username == null || username.isEmpty()) ? "NONE" : "BASIC";
+ VersionedFlowUtils.setParameterValue(externalFlow, "Schema Registry Authentication Type", authenticationType);
+ }
+ }
+
+ private void updateKafkaConnectionParameters(final VersionedExternalFlow externalFlow) {
+ final ConfigurationStep connectionStep = KafkaConnectionStep.KAFKA_CONNECTION_STEP;
+ final ConnectorPropertyGroup serverGroup = KafkaConnectionStep.KAFKA_SERVER_GROUP;
+
+ final String kafkaBrokers = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.KAFKA_BROKERS).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Bootstrap Servers", kafkaBrokers);
+
+ final String securityProtocol = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.SECURITY_PROTOCOL).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Security Protocol", securityProtocol);
+
+ if (securityProtocol.contains("SASL")) {
+ final String saslMechanism = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.SASL_MECHANISM).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka SASL Mechanism", saslMechanism);
+
+ final String username = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.USERNAME).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka SASL Username", username);
+
+ final String password = configContext.getProperty(connectionStep, serverGroup, KafkaConnectionStep.PASSWORD).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka SASL Password", password);
+ }
+ }
+
+ private void updateReaderWriter(final VersionedExternalFlow externalFlow) {
+ final VersionedProcessGroup rootGroup = externalFlow.getFlowContents();
+
+ final VersionedControllerService avroService = rootGroup.getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("AvroReader"))
+ .findFirst()
+ .orElseThrow();
+
+ final String kafkaDataFormat = configContext.getProperty(KafkaTopicsStep.STEP_NAME, KafkaTopicsStep.KAFKA_TOPICS_GROUP.getName(), KafkaTopicsStep.KAFKA_DATA_FORMAT.getName()).getValue();
+ if (!kafkaDataFormat.equalsIgnoreCase("JSON")) {
+ // Update ConsumeKafka processor to use Avro Reader
+ rootGroup.getProcessors().stream()
+ .filter(versionedProcessor -> versionedProcessor.getType().endsWith("ConsumeKafka"))
+ .findFirst()
+ .ifPresent(processor -> processor.getProperties().put("Record Reader", avroService.getIdentifier()));
+ }
+
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Data Format", kafkaDataFormat);
+
+ // TODO: Have this throw an Exception if GROUP does not contain the given property
+ final String s3DataFormat = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_DATA_FORMAT).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Data Format", s3DataFormat);
+ }
+
+ private void updateKafkaTopicsParameters(final VersionedExternalFlow externalFlow) {
+ final String groupName = KafkaTopicsStep.KAFKA_TOPICS_GROUP.getName();
+
+ final String topics = configContext.getProperty(KafkaTopicsStep.STEP_NAME, groupName, KafkaTopicsStep.TOPIC_NAMES.getName()).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Topic Names", topics);
+
+ final String groupId = configContext.getProperty(KafkaTopicsStep.STEP_NAME, groupName, KafkaTopicsStep.CONSUMER_GROUP_ID.getName()).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Consumer Group ID", groupId);
+
+ final String offsetReset = configContext.getProperty(KafkaTopicsStep.STEP_NAME, groupName, KafkaTopicsStep.OFFSET_RESET.getName()).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Kafka Auto Offset Reset", offsetReset);
+ }
+
+ private void updateS3Config(final VersionedExternalFlow externalFlow) {
+ final String region = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_REGION).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Region", region);
+
+ final String bucket = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_BUCKET).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Bucket", bucket);
+
+ final String prefix = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_PREFIX).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Prefix", prefix);
+
+ final String endpointOverrideUrl = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_DESTINATION_GROUP, S3Step.S3_ENDPOINT_OVERRIDE_URL).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Endpoint Override URL", endpointOverrideUrl);
+
+ final String authStrategy = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_CREDENTIALS_GROUP, S3Step.S3_AUTHENTICATION_STRATEGY).getValue();
+ if (authStrategy.equals(S3Step.DEFAULT_CREDENTIALS)) {
+ final VersionedControllerService credentialsService = externalFlow.getFlowContents().getControllerServices().stream()
+ .filter(service -> service.getType().endsWith("AWSCredentialsProviderControllerService"))
+ .findFirst()
+ .orElseThrow();
+
+ credentialsService.setProperties(Map.of("default-credentials", "true"));
+ } else {
+ final String accessKey = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_CREDENTIALS_GROUP, S3Step.S3_ACCESS_KEY_ID).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Access Key ID", accessKey);
+
+ final String secretKey = configContext.getProperty(S3Step.S3_STEP, S3Step.S3_CREDENTIALS_GROUP, S3Step.S3_SECRET_ACCESS_KEY).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "S3 Secret Access Key", secretKey);
+ }
+
+ final long mergeBytes = configContext.getProperty(S3Step.S3_STEP, S3Step.MERGE_GROUP, S3Step.TARGET_OBJECT_SIZE).asDataSize(DataUnit.B).longValue();
+ final String mergeSize = configContext.getProperty(S3Step.S3_STEP, S3Step.MERGE_GROUP, S3Step.TARGET_OBJECT_SIZE).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Target Object Size", mergeSize);
+
+ // Max Bin size will be either 10% more than target size or target size + 100MB, whichever is smaller
+ final long maxBinSize = (long) Math.min(mergeBytes + 100_000_000, mergeBytes * 1.1D);
+ VersionedFlowUtils.setParameterValue(externalFlow, "Maximum Object Size", maxBinSize + " B");
+
+ final String mergeLatency = configContext.getProperty(S3Step.S3_STEP, S3Step.MERGE_GROUP, S3Step.MERGE_LATENCY).getValue();
+ VersionedFlowUtils.setParameterValue(externalFlow, "Merge Latency", mergeLatency);
+ }
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaTopicsStep.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaTopicsStep.java
new file mode 100644
index 0000000000..643e4e836e
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaTopicsStep.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorPropertyDescriptor;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.PropertyType;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.List;
+
+public class KafkaTopicsStep {
+ public static final String STEP_NAME = "Kafka Topics";
+
+ public static final ConnectorPropertyDescriptor TOPIC_NAMES = new ConnectorPropertyDescriptor.Builder()
+ .name("Topic Names")
+ .description("A comma-separated list of Kafka topics to consume from.")
+ .required(true)
+ .type(PropertyType.STRING_LIST)
+ .allowableValuesFetchable(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor CONSUMER_GROUP_ID = new ConnectorPropertyDescriptor.Builder()
+ .name("Consumer Group ID")
+ .description("The consumer group ID to use when connecting to Kafka.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor OFFSET_RESET = new ConnectorPropertyDescriptor.Builder()
+ .name("Offset Reset")
+ .description("What to do when there is no initial offset in Kafka or if the current offset does not exist any more on the server.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .defaultValue("latest")
+ .allowableValues("earliest", "latest", "none")
+ .build();
+
+ public static final ConnectorPropertyDescriptor KAFKA_DATA_FORMAT = new ConnectorPropertyDescriptor.Builder()
+ .name("Kafka Data Format")
+ .description("The format of the data in Kafka topics.")
+ .required(true)
+ .type(PropertyType.STRING)
+ .defaultValue("Avro")
+ .allowableValues("Avro", "JSON")
+ .build();
+
+ public static final ConnectorPropertyGroup KAFKA_TOPICS_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Kafka Topics Configuration")
+ .description("Properties for configuring Kafka topics consumption.")
+ .properties(List.of(
+ TOPIC_NAMES,
+ CONSUMER_GROUP_ID,
+ OFFSET_RESET,
+ KAFKA_DATA_FORMAT
+ ))
+ .build();
+
+ public static final ConfigurationStep KAFKA_TOPICS_STEP = new ConfigurationStep.Builder()
+ .name(STEP_NAME)
+ .description("Kafka topics to consume from." )
+ .propertyGroups(List.of(
+ KAFKA_TOPICS_GROUP
+ ))
+ .build();
+
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java
new file mode 100644
index 0000000000..ad29115c4a
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import org.apache.nifi.components.connector.ConfigurationStep;
+import org.apache.nifi.components.connector.ConnectorPropertyDescriptor;
+import org.apache.nifi.components.connector.ConnectorPropertyGroup;
+import org.apache.nifi.components.connector.PropertyType;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.List;
+
+public class S3Step {
+ public static final String S3_STEP_NAME = "S3 Configuration";
+ public static final String ACCESS_KEY_ID_SECRET_KEY = "Access Key ID and Secret Key";
+ public static final String DEFAULT_CREDENTIALS = "Default AWS Credentials";
+
+
+ public static final ConnectorPropertyDescriptor S3_BUCKET = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Bucket")
+ .description("The name of the S3 bucket to write data to.")
+ .required(true)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_REGION = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Region")
+ .description("The AWS region where the S3 bucket is located.")
+ .allowableValuesFetchable(true)
+ .required(true)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_DATA_FORMAT = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Data Format")
+ .description("The format to use when writing data to S3.")
+ .required(true)
+ .defaultValue("JSON")
+ .allowableValues("Avro", "JSON")
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_PREFIX = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Prefix")
+ .description("An optional prefix to prepend to all object keys written to the S3 bucket.")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_ENDPOINT_OVERRIDE_URL = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Endpoint Override URL")
+ .description("An optional endpoint URL to use instead of the default AWS S3 endpoint. " +
+ "This can be used to connect to S3-compatible storage systems but should be left unset for connecting to S3.")
+ .required(false)
+ .addValidator(StandardValidators.URL_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor TARGET_OBJECT_SIZE = new ConnectorPropertyDescriptor.Builder()
+ .name("Target Object Size")
+ .description("The target size for each object written to S3. The connector will attempt to " +
+ "combine messages until this size is reached before writing an object to S3. Note that this size is approximate " +
+ "and may vary from object to object.")
+ .required(true)
+ .defaultValue("256 MB")
+ .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor MERGE_LATENCY = new ConnectorPropertyDescriptor.Builder()
+ .name("Merge Latency")
+ .description("The maximum amount of time to wait while merging messages before writing an object to S3. " +
+ "If this time is reached before the target object size is met, the current set of merged messages " +
+ "will be written to S3.")
+ .required(true)
+ .defaultValue("5 min")
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_AUTHENTICATION_STRATEGY = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Authentication Strategy")
+ .description("The authentication strategy to use when connecting to S3.")
+ .required(true)
+ .defaultValue(ACCESS_KEY_ID_SECRET_KEY)
+ .allowableValues(ACCESS_KEY_ID_SECRET_KEY, DEFAULT_CREDENTIALS)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_ACCESS_KEY_ID = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Access Key ID")
+ .description("The AWS Access Key ID used to authenticate to S3.")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(S3_AUTHENTICATION_STRATEGY, ACCESS_KEY_ID_SECRET_KEY)
+ .build();
+
+ public static final ConnectorPropertyDescriptor S3_SECRET_ACCESS_KEY = new ConnectorPropertyDescriptor.Builder()
+ .name("S3 Secret Access Key")
+ .description("The AWS Secret Access Key used to authenticate to S3.")
+ .required(true)
+ .type(PropertyType.SECRET)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dependsOn(S3_AUTHENTICATION_STRATEGY, ACCESS_KEY_ID_SECRET_KEY)
+ .build();
+
+
+ public static final ConnectorPropertyGroup S3_DESTINATION_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("S3 Destination Configuration")
+ .description("Properties required to connect to S3 and specify the target bucket.")
+ .properties(List.of(
+ S3_BUCKET,
+ S3_PREFIX,
+ S3_REGION,
+ S3_DATA_FORMAT,
+ S3_ENDPOINT_OVERRIDE_URL
+ ))
+ .build();
+
+ public static final ConnectorPropertyGroup S3_CREDENTIALS_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("S3 Credentials")
+ .description("Properties required to authenticate to S3.")
+ .properties(List.of(
+ S3_AUTHENTICATION_STRATEGY,
+ S3_ACCESS_KEY_ID,
+ S3_SECRET_ACCESS_KEY
+ ))
+ .build();
+
+ public static final ConnectorPropertyGroup MERGE_GROUP = new ConnectorPropertyGroup.Builder()
+ .name("Merge Configuration")
+ .description("Configuration for how data should be merged together before being persisted to S3.")
+ .properties(List.of(
+ TARGET_OBJECT_SIZE,
+ MERGE_LATENCY
+ ))
+ .build();
+
+ public static final ConfigurationStep S3_STEP = new ConfigurationStep.Builder()
+ .name(S3_STEP_NAME)
+ .description("Configure connection to S3 and target bucket details.")
+ .propertyGroups(List.of(
+ S3_DESTINATION_GROUP,
+ MERGE_GROUP,
+ S3_CREDENTIALS_GROUP
+ ))
+ .build();
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
similarity index 80%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
copy to nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
index 56740ff7c3..de5fe10a48 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector
@@ -12,6 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.nifi.test.processors.ModifiesClasspathProcessor
-org.apache.nifi.test.processors.ModifiesClasspathNoAnnotationProcessor
-org.apache.nifi.controller.service.mock.DummyProcessor
\ No newline at end of file
+
+org.apache.nifi.connectors.kafkas3.KafkaToS3
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/flows/Kafka_to_S3.json b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/flows/Kafka_to_S3.json
new file mode 100644
index 0000000000..5df6da47ae
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/resources/flows/Kafka_to_S3.json
@@ -0,0 +1,2105 @@
+{
+ "flowContents": {
+ "identifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "instanceIdentifier": "2c825ae7-019a-1000-f3b7-8275175b7b5b",
+ "name": "Kafka to S3",
+ "comments": "",
+ "position": {
+ "x": -355.0,
+ "y": -238.5
+ },
+ "processGroups": [],
+ "remoteProcessGroups": [],
+ "processors": [
+ {
+ "identifier": "4ce9587d-a171-3dff-bea8-bc67f4b356cd",
+ "instanceIdentifier": "40e02539-8a55-3a44-53d9-e9d41f7db23a",
+ "name": "MergeRecord",
+ "comments": "",
+ "position": {
+ "x": -250.0,
+ "y": -55.5
+ },
+ "type": "org.apache.nifi.processors.standard.MergeRecord",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "max-bin-age": "#{Merge Latency}",
+ "correlation-attribute-name": "kafka.topic",
+ "Attribute Strategy": "Keep Only Common Attributes",
+ "min-records": "1",
+ "max-bin-size": "#{Maximum Object size}",
+ "record-writer": "2384a9ae-311e-3d5f-983b-13f82174244c",
+ "merge-strategy": "Bin-Packing Algorithm",
+ "record-reader": "cd697ee5-9621-3680-a6b8-af3237472b5b",
+ "max-records": "1000000000",
+ "min-bin-size": "#{Target Object Size}",
+ "max.bin.count": "10"
+ },
+ "propertyDescriptors": {
+ "max-bin-age": {
+ "name": "max-bin-age",
+ "displayName": "Max Bin Age",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "correlation-attribute-name": {
+ "name": "correlation-attribute-name",
+ "displayName": "Correlation Attribute Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Attribute Strategy": {
+ "name": "Attribute Strategy",
+ "displayName": "Attribute Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "min-records": {
+ "name": "min-records",
+ "displayName": "Minimum Number of Records",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max-bin-size": {
+ "name": "max-bin-size",
+ "displayName": "Maximum Bin Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "record-writer": {
+ "name": "record-writer",
+ "displayName": "Record Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "merge-strategy": {
+ "name": "merge-strategy",
+ "displayName": "Merge Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "record-reader": {
+ "name": "record-reader",
+ "displayName": "Record Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max-records": {
+ "name": "max-records",
+ "displayName": "Maximum Number of Records",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "min-bin-size": {
+ "name": "min-bin-size",
+ "displayName": "Minimum Bin Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max.bin.count": {
+ "name": "max.bin.count",
+ "displayName": "Maximum Number of Bins",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "style": {},
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "executionNode": "ALL",
+ "penaltyDuration": "30 sec",
+ "yieldDuration": "1 sec",
+ "bulletinLevel": "WARN",
+ "runDurationMillis": 0,
+ "concurrentlySchedulableTaskCount": 1,
+ "autoTerminatedRelationships": [
+ "original",
+ "failure"
+ ],
+ "scheduledState": "ENABLED",
+ "retryCount": 10000,
+ "retriedRelationships": [
+ "failure"
+ ],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "maxBackoffPeriod": "10 mins",
+ "componentType": "PROCESSOR",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "e427a2b2-355c-3ede-9749-b3e4a8561d22",
+ "instanceIdentifier": "e048d449-7da8-30f3-8ff4-71c87c1d73db",
+ "name": "ConsumeKafka",
+ "comments": "",
+ "position": {
+ "x": -250.0,
+ "y": -268.5
+ },
+ "type": "org.apache.nifi.kafka.processors.ConsumeKafka",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-kafka-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Topics": "#{Topic Names}",
+ "Commit Offsets": "true",
+ "Header Name Pattern": null,
+ "Key Format": "byte-array",
+ "Key Record Reader": null,
+ "Message Demarcator": null,
+ "Record Reader": "919b2a35-4bc7-380c-b592-6ad2ffffbed5",
+ "Key Attribute Encoding": "utf-8",
+ "Max Uncommitted Size": null,
+ "Topic Format": "names",
+ "Header Encoding": "UTF-8",
+ "Max Uncommitted Time": "5 secs",
+ "Kafka Connection Service": "ccf048dc-004a-35be-8e4d-c7a763a30b02",
+ "Separate By Key": "false",
+ "Processing Strategy": "RECORD",
+ "Record Writer": "2384a9ae-311e-3d5f-983b-13f82174244c",
+ "Group ID": "#{Consumer Group ID}",
+ "auto.offset.reset": "#{Kafka Auto Offset Reset}",
+ "Output Strategy": "USE_VALUE"
+ },
+ "propertyDescriptors": {
+ "Topics": {
+ "name": "Topics",
+ "displayName": "Topics",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Commit Offsets": {
+ "name": "Commit Offsets",
+ "displayName": "Commit Offsets",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Header Name Pattern": {
+ "name": "Header Name Pattern",
+ "displayName": "Header Name Pattern",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Key Format": {
+ "name": "Key Format",
+ "displayName": "Key Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Key Record Reader": {
+ "name": "Key Record Reader",
+ "displayName": "Key Record Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Message Demarcator": {
+ "name": "Message Demarcator",
+ "displayName": "Message Demarcator",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Record Reader": {
+ "name": "Record Reader",
+ "displayName": "Record Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Key Attribute Encoding": {
+ "name": "Key Attribute Encoding",
+ "displayName": "Key Attribute Encoding",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max Uncommitted Size": {
+ "name": "Max Uncommitted Size",
+ "displayName": "Max Uncommitted Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Topic Format": {
+ "name": "Topic Format",
+ "displayName": "Topic Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Header Encoding": {
+ "name": "Header Encoding",
+ "displayName": "Header Encoding",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max Uncommitted Time": {
+ "name": "Max Uncommitted Time",
+ "displayName": "Max Uncommitted Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Kafka Connection Service": {
+ "name": "Kafka Connection Service",
+ "displayName": "Kafka Connection Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Separate By Key": {
+ "name": "Separate By Key",
+ "displayName": "Separate By Key",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Processing Strategy": {
+ "name": "Processing Strategy",
+ "displayName": "Processing Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Record Writer": {
+ "name": "Record Writer",
+ "displayName": "Record Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Group ID": {
+ "name": "Group ID",
+ "displayName": "Group ID",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "auto.offset.reset": {
+ "name": "auto.offset.reset",
+ "displayName": "Auto Offset Reset",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Output Strategy": {
+ "name": "Output Strategy",
+ "displayName": "Output Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "style": {},
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "executionNode": "ALL",
+ "penaltyDuration": "30 sec",
+ "yieldDuration": "1 sec",
+ "bulletinLevel": "WARN",
+ "runDurationMillis": 0,
+ "concurrentlySchedulableTaskCount": 1,
+ "autoTerminatedRelationships": [
+ "parse failure"
+ ],
+ "scheduledState": "ENABLED",
+ "retryCount": 10,
+ "retriedRelationships": [],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "maxBackoffPeriod": "10 mins",
+ "componentType": "PROCESSOR",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "501f25bb-cef8-328f-bfc2-fe58ad12e7b6",
+ "instanceIdentifier": "3de30158-1d3e-38d9-6e68-9578471aa26d",
+ "name": "PutS3Object",
+ "comments": "",
+ "position": {
+ "x": -250.0,
+ "y": 169.5
+ },
+ "type": "org.apache.nifi.processors.aws.s3.PutS3Object",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "FullControl User List": "${s3.permissions.full.users}",
+ "Owner": "${s3.owner}",
+ "proxy-configuration-service": null,
+ "Resource Transfer Source": "FLOWFILE_CONTENT",
+ "Cache Control": null,
+ "Endpoint Override URL": "#{S3 Endpoint Override URL}",
+ "File Resource Service": null,
+ "Multipart Upload Max Age Threshold": "7 days",
+ "Object Tags Prefix": null,
+ "Signer Override": "Default Signature",
+ "Content Type": "${mime.type}",
+ "Canned ACL": "${s3.permissions.cannedacl}",
+ "Use Path Style Access": "false",
+ "Encryption Service": null,
+ "Write ACL User List": "${s3.permissions.writeacl.users}",
+ "Use Chunked Encoding": "true",
+ "Read ACL User List": "${s3.permissions.readacl.users}",
+ "Content Disposition": null,
+ "Storage Class": "Standard",
+ "Multipart Part Size": "5 GB",
+ "Remove Tag Prefix": "false",
+ "Object Key": "${#{S3 Prefix}:append(\"/\"):replaceAll('^/$', '')}${filename}",
+ "AWS Credentials Provider service": "3f5da96c-9c54-39aa-8c84-e8098a44dfab",
+ "Multipart Threshold": "5 GB",
+ "Temporary Directory Multipart State": "${java.io.tmpdir}",
+ "Custom Signer Module Location": null,
+ "SSL Context Service": null,
+ "Server Side Encryption": "None",
+ "Bucket": "#{S3 Bucket}",
+ "Multipart Upload AgeOff Interval": "60 min",
+ "Write Permission User List": "${s3.permissions.write.users}",
+ "Communications Timeout": "30 secs",
+ "Region": "#{S3 Region}",
+ "Read Permission User List": "${s3.permissions.read.users}",
+ "Custom Signer Class Name": null,
+ "Expiration Time Rule": null
+ },
+ "propertyDescriptors": {
+ "FullControl User List": {
+ "name": "FullControl User List",
+ "displayName": "FullControl User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Owner": {
+ "name": "Owner",
+ "displayName": "Owner",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "proxy-configuration-service": {
+ "name": "proxy-configuration-service",
+ "displayName": "Proxy Configuration Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Resource Transfer Source": {
+ "name": "Resource Transfer Source",
+ "displayName": "Resource Transfer Source",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Cache Control": {
+ "name": "Cache Control",
+ "displayName": "Cache Control",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Endpoint Override URL": {
+ "name": "Endpoint Override URL",
+ "displayName": "Endpoint Override URL",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "File Resource Service": {
+ "name": "File Resource Service",
+ "displayName": "File Resource Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Upload Max Age Threshold": {
+ "name": "Multipart Upload Max Age Threshold",
+ "displayName": "Multipart Upload Max Age Threshold",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Object Tags Prefix": {
+ "name": "Object Tags Prefix",
+ "displayName": "Object Tags Prefix",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Signer Override": {
+ "name": "Signer Override",
+ "displayName": "Signer Override",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Content Type": {
+ "name": "Content Type",
+ "displayName": "Content Type",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Canned ACL": {
+ "name": "Canned ACL",
+ "displayName": "Canned ACL",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Use Path Style Access": {
+ "name": "Use Path Style Access",
+ "displayName": "Use Path Style Access",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Encryption Service": {
+ "name": "Encryption Service",
+ "displayName": "Encryption Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Write ACL User List": {
+ "name": "Write ACL User List",
+ "displayName": "Write ACL User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Use Chunked Encoding": {
+ "name": "Use Chunked Encoding",
+ "displayName": "Use Chunked Encoding",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Read ACL User List": {
+ "name": "Read ACL User List",
+ "displayName": "Read ACL User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Content Disposition": {
+ "name": "Content Disposition",
+ "displayName": "Content Disposition",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Storage Class": {
+ "name": "Storage Class",
+ "displayName": "Storage Class",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Part Size": {
+ "name": "Multipart Part Size",
+ "displayName": "Multipart Part Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Remove Tag Prefix": {
+ "name": "Remove Tag Prefix",
+ "displayName": "Remove Tag Prefix",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Object Key": {
+ "name": "Object Key",
+ "displayName": "Object Key",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "AWS Credentials Provider service": {
+ "name": "AWS Credentials Provider service",
+ "displayName": "AWS Credentials Provider service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Threshold": {
+ "name": "Multipart Threshold",
+ "displayName": "Multipart Threshold",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Temporary Directory Multipart State": {
+ "name": "Temporary Directory Multipart State",
+ "displayName": "Temporary Directory Multipart State",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Custom Signer Module Location": {
+ "name": "Custom Signer Module Location",
+ "displayName": "Custom Signer Module Location",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false,
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE",
+ "DIRECTORY"
+ ]
+ }
+ },
+ "SSL Context Service": {
+ "name": "SSL Context Service",
+ "displayName": "SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Server Side Encryption": {
+ "name": "Server Side Encryption",
+ "displayName": "Server Side Encryption",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Bucket": {
+ "name": "Bucket",
+ "displayName": "Bucket",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Multipart Upload AgeOff Interval": {
+ "name": "Multipart Upload AgeOff Interval",
+ "displayName": "Multipart Upload AgeOff Interval",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Write Permission User List": {
+ "name": "Write Permission User List",
+ "displayName": "Write Permission User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Communications Timeout": {
+ "name": "Communications Timeout",
+ "displayName": "Communications Timeout",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Region": {
+ "name": "Region",
+ "displayName": "Region",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Read Permission User List": {
+ "name": "Read Permission User List",
+ "displayName": "Read Permission User List",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Custom Signer Class Name": {
+ "name": "Custom Signer Class Name",
+ "displayName": "Custom Signer Class Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Expiration Time Rule": {
+ "name": "Expiration Time Rule",
+ "displayName": "Expiration Time Rule",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "style": {},
+ "schedulingPeriod": "0 sec",
+ "schedulingStrategy": "TIMER_DRIVEN",
+ "executionNode": "ALL",
+ "penaltyDuration": "30 sec",
+ "yieldDuration": "1 sec",
+ "bulletinLevel": "WARN",
+ "runDurationMillis": 0,
+ "concurrentlySchedulableTaskCount": 1,
+ "autoTerminatedRelationships": [
+ "success",
+ "failure"
+ ],
+ "scheduledState": "ENABLED",
+ "retryCount": 100000,
+ "retriedRelationships": [
+ "failure"
+ ],
+ "backoffMechanism": "PENALIZE_FLOWFILE",
+ "maxBackoffPeriod": "10 mins",
+ "componentType": "PROCESSOR",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ }
+ ],
+ "inputPorts": [],
+ "outputPorts": [],
+ "connections": [
+ {
+ "identifier": "4bd001fc-7fa6-3c34-b297-bae4987889ad",
+ "instanceIdentifier": "d6a7b460-571b-39b3-b55f-7d82f62b397a",
+ "name": "",
+ "source": {
+ "id": "e427a2b2-355c-3ede-9749-b3e4a8561d22",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "ConsumeKafka",
+ "comments": "",
+ "instanceIdentifier": "e048d449-7da8-30f3-8ff4-71c87c1d73db"
+ },
+ "destination": {
+ "id": "4ce9587d-a171-3dff-bea8-bc67f4b356cd",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "MergeRecord",
+ "comments": "",
+ "instanceIdentifier": "40e02539-8a55-3a44-53d9-e9d41f7db23a"
+ },
+ "labelIndex": 0,
+ "zIndex": 1,
+ "selectedRelationships": [
+ "success"
+ ],
+ "backPressureObjectThreshold": 10000,
+ "backPressureDataSizeThreshold": "1 GB",
+ "flowFileExpiration": "0 sec",
+ "prioritizers": [],
+ "bends": [],
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "partitioningAttribute": "",
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "componentType": "CONNECTION",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "08803783-a109-3362-9177-0f2016086a2a",
+ "instanceIdentifier": "45276df0-1076-39a8-7d84-097b7f593486",
+ "name": "",
+ "source": {
+ "id": "4ce9587d-a171-3dff-bea8-bc67f4b356cd",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "MergeRecord",
+ "comments": "",
+ "instanceIdentifier": "40e02539-8a55-3a44-53d9-e9d41f7db23a"
+ },
+ "destination": {
+ "id": "501f25bb-cef8-328f-bfc2-fe58ad12e7b6",
+ "type": "PROCESSOR",
+ "groupId": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b",
+ "name": "PutS3Object",
+ "comments": "",
+ "instanceIdentifier": "3de30158-1d3e-38d9-6e68-9578471aa26d"
+ },
+ "labelIndex": 0,
+ "zIndex": 2,
+ "selectedRelationships": [
+ "merged"
+ ],
+ "backPressureObjectThreshold": 10000,
+ "backPressureDataSizeThreshold": "1 GB",
+ "flowFileExpiration": "0 sec",
+ "prioritizers": [],
+ "bends": [],
+ "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
+ "partitioningAttribute": "",
+ "loadBalanceCompression": "DO_NOT_COMPRESS",
+ "componentType": "CONNECTION",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ }
+ ],
+ "labels": [],
+ "funnels": [],
+ "controllerServices": [
+ {
+ "identifier": "cd697ee5-9621-3680-a6b8-af3237472b5b",
+ "instanceIdentifier": "2cdfec74-019a-1000-2786-edf80b919dfc",
+ "name": "Merge JSON Reader",
+ "comments": "",
+ "type": "org.apache.nifi.json.JsonTreeReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": null,
+ "Schema Text": "${avro.schema}",
+ "Max String Length": "20 MB",
+ "schema-application-strategy": "SELECTED_PART",
+ "Schema Registry": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "Schema Access Strategy": "schema-name",
+ "Schema Name": "${schema.name}",
+ "Timestamp Format": null,
+ "schema-inference-cache": null,
+ "Date Format": null,
+ "starting-field-strategy": "ROOT_NODE",
+ "Schema Version": null,
+ "starting-field-name": null,
+ "Time Format": null,
+ "Allow Comments": "false"
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max String Length": {
+ "name": "Max String Length",
+ "displayName": "Max String Length",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-application-strategy": {
+ "name": "schema-application-strategy",
+ "displayName": "Schema Application Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Timestamp Format": {
+ "name": "Timestamp Format",
+ "displayName": "Timestamp Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-inference-cache": {
+ "name": "schema-inference-cache",
+ "displayName": "Schema Inference Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Date Format": {
+ "name": "Date Format",
+ "displayName": "Date Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-strategy": {
+ "name": "starting-field-strategy",
+ "displayName": "Starting Field Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-name": {
+ "name": "starting-field-name",
+ "displayName": "Starting Field Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Time Format": {
+ "name": "Time Format",
+ "displayName": "Time Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Allow Comments": {
+ "name": "Allow Comments",
+ "displayName": "Allow Comments",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordReaderFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "b01590c6-5149-3c50-9d16-3a8e7b1bcdf6",
+ "instanceIdentifier": "2f0a61a3-b66d-3969-09d7-df35dd0b2740",
+ "name": "AvroRecordSetWriter",
+ "comments": "",
+ "type": "org.apache.nifi.avro.AvroRecordSetWriter",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": null,
+ "Schema Text": "${avro.schema}",
+ "Schema Registry": null,
+ "Schema Access Strategy": "inherit-record-schema",
+ "Schema Name": "${schema.name}",
+ "compression-format": "NONE",
+ "Schema Write Strategy": "avro-embedded",
+ "cache-size": "1000",
+ "Schema Cache": null,
+ "Schema Version": null,
+ "encoder-pool-size": "32",
+ "Schema Reference Writer": null
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "compression-format": {
+ "name": "compression-format",
+ "displayName": "Compression Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Write Strategy": {
+ "name": "Schema Write Strategy",
+ "displayName": "Schema Write Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "cache-size": {
+ "name": "cache-size",
+ "displayName": "Cache Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Cache": {
+ "name": "Schema Cache",
+ "displayName": "Schema Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "encoder-pool-size": {
+ "name": "encoder-pool-size",
+ "displayName": "Encoder Pool Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Writer": {
+ "name": "Schema Reference Writer",
+ "displayName": "Schema Reference Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordSetWriterFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "919b2a35-4bc7-380c-b592-6ad2ffffbed5",
+ "instanceIdentifier": "c169a376-4b21-3062-e8fb-1d243dd0e5df",
+ "name": "Kafka JSON Reader",
+ "comments": "",
+ "type": "org.apache.nifi.json.JsonTreeReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": "ecba6fe5-6beb-3b68-bad9-163a3f14c4c8",
+ "Schema Text": "${avro.schema}",
+ "Max String Length": "20 MB",
+ "schema-application-strategy": "SELECTED_PART",
+ "Schema Registry": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "Schema Access Strategy": "schema-reference-reader",
+ "Schema Name": "${schema.name}",
+ "Timestamp Format": null,
+ "schema-inference-cache": null,
+ "Date Format": null,
+ "starting-field-strategy": "ROOT_NODE",
+ "Schema Version": null,
+ "starting-field-name": null,
+ "Time Format": null,
+ "Allow Comments": "false"
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Max String Length": {
+ "name": "Max String Length",
+ "displayName": "Max String Length",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-application-strategy": {
+ "name": "schema-application-strategy",
+ "displayName": "Schema Application Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Timestamp Format": {
+ "name": "Timestamp Format",
+ "displayName": "Timestamp Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "schema-inference-cache": {
+ "name": "schema-inference-cache",
+ "displayName": "Schema Inference Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Date Format": {
+ "name": "Date Format",
+ "displayName": "Date Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-strategy": {
+ "name": "starting-field-strategy",
+ "displayName": "Starting Field Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "starting-field-name": {
+ "name": "starting-field-name",
+ "displayName": "Starting Field Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Time Format": {
+ "name": "Time Format",
+ "displayName": "Time Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Allow Comments": {
+ "name": "Allow Comments",
+ "displayName": "Allow Comments",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordReaderFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "174d0040-161d-39dc-8da9-0f127fc46373",
+ "instanceIdentifier": "bf48cf99-1cae-375b-bfe1-bd9aa7368984",
+ "name": "AvroReader",
+ "comments": "",
+ "type": "org.apache.nifi.avro.AvroReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": "ecba6fe5-6beb-3b68-bad9-163a3f14c4c8",
+ "Schema Text": "${avro.schema}",
+ "cache-size": "1000",
+ "Schema Version": null,
+ "Schema Registry": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "Schema Access Strategy": "schema-reference-reader",
+ "Schema Name": "${schema.name}"
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "cache-size": {
+ "name": "cache-size",
+ "displayName": "Cache Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordReaderFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "3f5da96c-9c54-39aa-8c84-e8098a44dfab",
+ "instanceIdentifier": "7ea5a25b-89be-3652-9778-e30a69f84806",
+ "name": "AWSCredentialsProviderControllerService",
+ "comments": "",
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Assume Role STS Region": "us-west-2",
+ "Assume Role Proxy Configuration Service": null,
+ "Access Key ID": "#{S3 Access Key ID}",
+ "Assume Role ARN": null,
+ "Assume Role Session Time": "3600",
+ "Assume Role Session Name": null,
+ "Custom Signer Module Location": null,
+ "Use Default Credentials": "false",
+ "Assume Role External ID": null,
+ "Assume Role STS Endpoint Override": null,
+ "Profile Name": null,
+ "Credentials File": null,
+ "Custom Signer Class Name": null,
+ "Assume Role STS Signer Override": "Default Signature",
+ "Use Anonymous Credentials": "false",
+ "Secret Access Key": "#{S3 Secret Access Key}",
+ "Assume Role SSL Context Service": null
+ },
+ "propertyDescriptors": {
+ "Assume Role STS Region": {
+ "name": "Assume Role STS Region",
+ "displayName": "Assume Role STS Region",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role Proxy Configuration Service": {
+ "name": "Assume Role Proxy Configuration Service",
+ "displayName": "Assume Role Proxy Configuration Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Access Key ID": {
+ "name": "Access Key ID",
+ "displayName": "Access Key ID",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ },
+ "Assume Role ARN": {
+ "name": "Assume Role ARN",
+ "displayName": "Assume Role ARN",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role Session Time": {
+ "name": "Assume Role Session Time",
+ "displayName": "Assume Role Session Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role Session Name": {
+ "name": "Assume Role Session Name",
+ "displayName": "Assume Role Session Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Custom Signer Module Location": {
+ "name": "Custom Signer Module Location",
+ "displayName": "Custom Signer Module Location",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false,
+ "resourceDefinition": {
+ "cardinality": "MULTIPLE",
+ "resourceTypes": [
+ "FILE",
+ "DIRECTORY"
+ ]
+ }
+ },
+ "Use Default Credentials": {
+ "name": "Use Default Credentials",
+ "displayName": "Use Default Credentials",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role External ID": {
+ "name": "Assume Role External ID",
+ "displayName": "Assume Role External ID",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role STS Endpoint Override": {
+ "name": "Assume Role STS Endpoint Override",
+ "displayName": "Assume Role STS Endpoint Override",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Profile Name": {
+ "name": "Profile Name",
+ "displayName": "Profile Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Credentials File": {
+ "name": "Credentials File",
+ "displayName": "Credentials File",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false,
+ "resourceDefinition": {
+ "cardinality": "SINGLE",
+ "resourceTypes": [
+ "FILE"
+ ]
+ }
+ },
+ "Custom Signer Class Name": {
+ "name": "Custom Signer Class Name",
+ "displayName": "Custom Signer Class Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Assume Role STS Signer Override": {
+ "name": "Assume Role STS Signer Override",
+ "displayName": "Assume Role STS Signer Override",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Use Anonymous Credentials": {
+ "name": "Use Anonymous Credentials",
+ "displayName": "Use Anonymous Credentials",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Secret Access Key": {
+ "name": "Secret Access Key",
+ "displayName": "Secret Access Key",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ },
+ "Assume Role SSL Context Service": {
+ "name": "Assume Role SSL Context Service",
+ "displayName": "Assume Role SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-service-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ },
+ {
+ "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-aws-service-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "2384a9ae-311e-3d5f-983b-13f82174244c",
+ "instanceIdentifier": "d02eb508-0c95-31df-b86b-e1adadd77845",
+ "name": "JsonRecordSetWriter",
+ "comments": "",
+ "type": "org.apache.nifi.json.JsonRecordSetWriter",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-record-serialization-services-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "Schema Branch": null,
+ "Schema Reference Reader": null,
+ "Schema Text": "${avro.schema}",
+ "Allow Scientific Notation": "false",
+ "compression-level": "1",
+ "Schema Registry": null,
+ "Schema Access Strategy": "inherit-record-schema",
+ "Schema Name": "${schema.name}",
+ "Timestamp Format": null,
+ "Date Format": null,
+ "Pretty Print JSON": "false",
+ "compression-format": "none",
+ "Schema Write Strategy": "schema-name",
+ "suppress-nulls": "never-suppress",
+ "output-grouping": "output-oneline",
+ "Schema Cache": null,
+ "Schema Version": null,
+ "Time Format": null,
+ "Schema Reference Writer": null
+ },
+ "propertyDescriptors": {
+ "Schema Branch": {
+ "name": "Schema Branch",
+ "displayName": "Schema Branch",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Reader": {
+ "name": "Schema Reference Reader",
+ "displayName": "Schema Reference Reader",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Text": {
+ "name": "Schema Text",
+ "displayName": "Schema Text",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Allow Scientific Notation": {
+ "name": "Allow Scientific Notation",
+ "displayName": "Allow Scientific Notation",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "compression-level": {
+ "name": "compression-level",
+ "displayName": "Compression Level",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry": {
+ "name": "Schema Registry",
+ "displayName": "Schema Registry",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Access Strategy": {
+ "name": "Schema Access Strategy",
+ "displayName": "Schema Access Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Name": {
+ "name": "Schema Name",
+ "displayName": "Schema Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Timestamp Format": {
+ "name": "Timestamp Format",
+ "displayName": "Timestamp Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Date Format": {
+ "name": "Date Format",
+ "displayName": "Date Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Pretty Print JSON": {
+ "name": "Pretty Print JSON",
+ "displayName": "Pretty Print JSON",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "compression-format": {
+ "name": "compression-format",
+ "displayName": "Compression Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Write Strategy": {
+ "name": "Schema Write Strategy",
+ "displayName": "Schema Write Strategy",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "suppress-nulls": {
+ "name": "suppress-nulls",
+ "displayName": "Suppress Null Values",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "output-grouping": {
+ "name": "output-grouping",
+ "displayName": "Output Grouping",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Cache": {
+ "name": "Schema Cache",
+ "displayName": "Schema Cache",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Version": {
+ "name": "Schema Version",
+ "displayName": "Schema Version",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Time Format": {
+ "name": "Time Format",
+ "displayName": "Time Format",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Reference Writer": {
+ "name": "Schema Reference Writer",
+ "displayName": "Schema Reference Writer",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.serialization.RecordSetWriterFactory",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "34d29160-0744-373c-a1c7-39c548e7e6b1",
+ "instanceIdentifier": "e0f7c153-e070-369b-451d-745b53ebe26a",
+ "name": "ConfluentSchemaRegistry",
+ "comments": "",
+ "type": "org.apache.nifi.confluent.schemaregistry.ConfluentSchemaRegistry",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-confluent-platform-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "SSL Context Service": null,
+ "Username": "#{Schema Registry Username}",
+ "Schema Registry URLs": "#{Schema Registry URLs}",
+ "Communications Timeout": "30 secs",
+ "Cache Size": "1000",
+ "Authentication Type": "#{Schema Registry Authentication Type}",
+ "Cache Expiration": "1 hour",
+ "Password": "#{Schema Registry Password}"
+ },
+ "propertyDescriptors": {
+ "SSL Context Service": {
+ "name": "SSL Context Service",
+ "displayName": "SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Username": {
+ "name": "Username",
+ "displayName": "Username",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Schema Registry URLs": {
+ "name": "Schema Registry URLs",
+ "displayName": "Schema Registry URLs",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Communications Timeout": {
+ "name": "Communications Timeout",
+ "displayName": "Communications Timeout",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Cache Size": {
+ "name": "Cache Size",
+ "displayName": "Cache Size",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Authentication Type": {
+ "name": "Authentication Type",
+ "displayName": "Authentication Type",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Cache Expiration": {
+ "name": "Cache Expiration",
+ "displayName": "Cache Expiration",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "Password": {
+ "name": "Password",
+ "displayName": "Password",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.schemaregistry.services.SchemaRegistry",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "ecba6fe5-6beb-3b68-bad9-163a3f14c4c8",
+ "instanceIdentifier": "da19419b-4821-315c-74b2-03d97cf411dc",
+ "name": "ConfluentEncodedSchemaReferenceReader",
+ "comments": "",
+ "type": "org.apache.nifi.confluent.schemaregistry.ConfluentEncodedSchemaReferenceReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-confluent-platform-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {},
+ "propertyDescriptors": {},
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.schemaregistry.services.SchemaReferenceReader",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-standard-services-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ },
+ {
+ "identifier": "ccf048dc-004a-35be-8e4d-c7a763a30b02",
+ "instanceIdentifier": "5549d8f6-48eb-3677-821e-292b9b7e73d2",
+ "name": "Kafka3ConnectionService",
+ "comments": "",
+ "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-kafka-3-service-nar",
+ "version": "2.7.0-SNAPSHOT"
+ },
+ "properties": {
+ "ack.wait.time": "10 sec",
+ "sasl.username": "#{Kafka SASL Username}",
+ "max.poll.records": "20000",
+ "bootstrap.servers": "#{Kafka Bootstrap Servers}",
+ "sasl.kerberos.service.name": null,
+ "security.protocol": "#{Kafka Security Protocol}",
+ "SSL Context Service": null,
+ "sasl.mechanism": "#{Kafka SASL Mechanism}",
+ "isolation.level": "read_committed",
+ "oauth2-access-token-provider-service": null,
+ "max.block.ms": "10 sec",
+ "kerberos-user-service": null,
+ "sasl.password": "#{Kafka SASL Password}",
+ "default.api.timeout.ms": "60 sec"
+ },
+ "propertyDescriptors": {
+ "ack.wait.time": {
+ "name": "ack.wait.time",
+ "displayName": "Acknowledgment Wait Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.username": {
+ "name": "sasl.username",
+ "displayName": "SASL Username",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max.poll.records": {
+ "name": "max.poll.records",
+ "displayName": "Max Poll Records",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "bootstrap.servers": {
+ "name": "bootstrap.servers",
+ "displayName": "Bootstrap Servers",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.kerberos.service.name": {
+ "name": "sasl.kerberos.service.name",
+ "displayName": "Kerberos Service Name",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "security.protocol": {
+ "name": "security.protocol",
+ "displayName": "Security Protocol",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "SSL Context Service": {
+ "name": "SSL Context Service",
+ "displayName": "SSL Context Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.mechanism": {
+ "name": "sasl.mechanism",
+ "displayName": "SASL Mechanism",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "isolation.level": {
+ "name": "isolation.level",
+ "displayName": "Transaction Isolation Level",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "oauth2-access-token-provider-service": {
+ "name": "oauth2-access-token-provider-service",
+ "displayName": "OAuth2 Access Token Provider Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "max.block.ms": {
+ "name": "max.block.ms",
+ "displayName": "Max Metadata Wait Time",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "kerberos-user-service": {
+ "name": "kerberos-user-service",
+ "displayName": "Kerberos User Service",
+ "identifiesControllerService": true,
+ "sensitive": false,
+ "dynamic": false
+ },
+ "sasl.password": {
+ "name": "sasl.password",
+ "displayName": "SASL Password",
+ "identifiesControllerService": false,
+ "sensitive": true,
+ "dynamic": false
+ },
+ "default.api.timeout.ms": {
+ "name": "default.api.timeout.ms",
+ "displayName": "Client Timeout",
+ "identifiesControllerService": false,
+ "sensitive": false,
+ "dynamic": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService",
+ "bundle": {
+ "group": "org.apache.nifi",
+ "artifact": "nifi-kafka-service-api-nar",
+ "version": "2.7.0-SNAPSHOT"
+ }
+ }
+ ],
+ "scheduledState": "DISABLED",
+ "bulletinLevel": "WARN",
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "b3a5c0de-2130-3ba9-942b-9bae5735bd4b"
+ }
+ ],
+ "parameterContextName": "Kafka to S3",
+ "defaultFlowFileExpiration": "0 sec",
+ "defaultBackPressureObjectThreshold": 10000,
+ "defaultBackPressureDataSizeThreshold": "1 GB",
+ "scheduledState": "ENABLED",
+ "executionEngine": "INHERITED",
+ "maxConcurrentTasks": 1,
+ "statelessFlowTimeout": "1 min",
+ "componentType": "PROCESS_GROUP",
+ "flowFileConcurrency": "UNBOUNDED",
+ "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE"
+ },
+ "externalControllerServices": {},
+ "parameterContexts": {
+ "Kafka to S3": {
+ "name": "Kafka to S3",
+ "parameters": [
+ {
+ "name": "S3 Secret Access Key",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Schema Registry Password",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Consumer Group ID",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Kafka SASL Mechanism",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "SCRAM-SHA-512"
+ },
+ {
+ "name": "Kafka Data Format",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "JSON"
+ },
+ {
+ "name": "Schema Registry Username",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "S3 Data Format",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "JSON"
+ },
+ {
+ "name": "Schema Registry URLs",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "http://localhost:8081"
+ },
+ {
+ "name": "Merge Latency",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "10 mins"
+ },
+ {
+ "name": "S3 Endpoint Override URL",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Schema Registry Authentication Type",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "BASIC"
+ },
+ {
+ "name": "Kafka Security Protocol",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "SASL_SSL"
+ },
+ {
+ "name": "Maximum Object size",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "300 MB"
+ },
+ {
+ "name": "S3 Region",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "us-west-2"
+ },
+ {
+ "name": "Kafka SASL Password",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Kafka Bootstrap Servers",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Kafka Auto Offset Reset",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "latest"
+ },
+ {
+ "name": "S3 Bucket",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "Kafka SASL Username",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "S3 Access Key ID",
+ "description": "",
+ "sensitive": true,
+ "provided": false
+ },
+ {
+ "name": "Target Object Size",
+ "description": "",
+ "sensitive": false,
+ "provided": false,
+ "value": "256 MB"
+ },
+ {
+ "name": "Topic Names",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ },
+ {
+ "name": "S3 Prefix",
+ "description": "",
+ "sensitive": false,
+ "provided": false
+ }
+ ],
+ "inheritedParameterContexts": [],
+ "description": "",
+ "componentType": "PARAMETER_CONTEXT"
+ }
+ },
+ "flowEncodingVersion": "1.0",
+ "parameterProviders": {},
+ "latest": false
+}
\ No newline at end of file
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml
new file mode 100644
index 0000000000..c36971572c
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml
@@ -0,0 +1,226 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-integration-tests</artifactId>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>https://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-api</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Test Containers for Kafka and S3 -->
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers-kafka</artifactId>
+ <version>2.0.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>localstack</artifactId>
+ <version>1.20.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Kafka client for producing test data -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>3.9.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Confluent Schema Registry and Avro dependencies for testing -->
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-avro-serializer</artifactId>
+ <version>7.8.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>7.8.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.11.3</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- AWS SDK for S3 verification in tests -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <version>2.36.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>url-connection-client</artifactId>
+ <version>2.36.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Necessary NAR dependencies -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-3-service-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-service-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-service-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-shared-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-services-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-confluent-platform-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+
+ <!-- NARs required by framework -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-jetty-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-provenance-repository-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-mock-server-nar</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <!-- Copy nars into libDir for test harness -->
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-nar</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/libDir</outputDirectory>
+ <includeTypes>nar</includeTypes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/src/test/java/org/apache/nifi/connectors/kafkas3/KafkaToS3IT.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/src/test/java/org/apache/nifi/connectors/kafkas3/KafkaToS3IT.java
new file mode 100644
index 0000000000..77b748c74e
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/src/test/java/org/apache/nifi/connectors/kafkas3/KafkaToS3IT.java
@@ -0,0 +1,742 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.connectors.kafkas3;
+
+import io.confluent.kafka.serializers.KafkaAvroSerializer;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.config.SaslConfigs;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.FlowUpdateException;
+import org.apache.nifi.components.connector.PropertyGroupConfiguration;
+import org.apache.nifi.mock.connector.StandardConnectorTestRunner;
+import org.apache.nifi.mock.connector.server.ConnectorConfigVerificationResult;
+import org.apache.nifi.mock.connector.server.ConnectorTestRunner;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.kafka.ConfluentKafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class KafkaToS3IT {
+
+ private static ConnectorTestRunner runner;
+ private static Network network;
+ private static ConfluentKafkaContainer kafkaContainer;
+ private static GenericContainer<?> schemaRegistryContainer;
+ private static LocalStackContainer localStackContainer;
+ private static S3Client s3Client;
+
+ private static final String SCRAM_USERNAME = "testuser";
+ private static final String SCRAM_PASSWORD = "testpassword";
+
+ private static final String S3_REGION = "us-west-2";
+
+ // JAAS configuration for Kafka broker SASL/PLAIN authentication.
+ // The 'username' and 'password' fields are credentials the broker uses for inter-broker communication.
+ // The 'user_<username>="<password>"' entries define client users that can authenticate to this broker.
+ // In this setup:
+ // - Broker uses 'admin' / 'admin-secret' for inter-broker communication (though we use PLAINTEXT for that)
+ // - Clients can authenticate using 'testuser' / 'testpassword' on the SASL listener with PLAIN mechanism
+ private static final String JAAS_CONFIG_CONTENT = """
+ KafkaServer {
+ org.apache.kafka.common.security.plain.PlainLoginModule required
+ username="admin"
+ password="admin-secret"
+ user_%s="%s";
+ };
+ """.formatted(SCRAM_USERNAME, SCRAM_PASSWORD);
+
+
+ @BeforeAll
+ public static void setupTestContainers() {
+ network = Network.newNetwork();
+
+ kafkaContainer = new ConfluentKafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.8.0"))
+ .withNetwork(network)
+ .withNetworkAliases("kafka")
+ .withStartupTimeout(Duration.ofSeconds(10))
+ .withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SASL:SASL_PLAINTEXT")
+ .withEnv("KAFKA_LISTENERS", "CONTROLLER://0.0.0.0:9094,BROKER://0.0.0.0:9092,PLAINTEXT://0.0.0.0:19092,SASL://0.0.0.0:9093")
+ .withEnv("KAFKA_ADVERTISED_LISTENERS", "BROKER://kafka:9092,PLAINTEXT://kafka:19092,SASL://localhost:9093")
+ .withEnv("KAFKA_CONTROLLER_LISTENER_NAMES", "CONTROLLER")
+ .withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER")
+ .withEnv("KAFKA_SASL_ENABLED_MECHANISMS", "PLAIN")
+ .withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0")
+ .withEnv("KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS", "1000")
+ .withEnv("KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS", "60000")
+ .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
+ .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1")
+ .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1")
+ .withEnv("KAFKA_OPTS", "-Djava.security.auth.login.config=/tmp/kafka_jaas.conf")
+ .withCommand(
+ "sh", "-c",
+ "echo '" + JAAS_CONFIG_CONTENT + "' > /tmp/kafka_jaas.conf && " +
+ "/etc/confluent/docker/run"
+ );
+
+ kafkaContainer.setPortBindings(List.of("9093:9093"));
+ kafkaContainer.start();
+
+ schemaRegistryContainer = new GenericContainer<>(DockerImageName.parse("confluentinc/cp-schema-registry:7.8.0"))
+ .withNetwork(network)
+ .withExposedPorts(8081)
+ .withEnv("SCHEMA_REGISTRY_HOST_NAME", "schema-registry")
+ .withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "PLAINTEXT://kafka:19092")
+ .withEnv("SCHEMA_REGISTRY_LISTENERS", "http://0.0.0.0:8081")
+ .withStartupTimeout(Duration.ofSeconds(60))
+ .waitingFor(Wait.forHttp("/subjects").forStatusCode(200))
+ .dependsOn(kafkaContainer);
+
+ schemaRegistryContainer.start();
+
+ localStackContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.1.0"))
+ .withServices(LocalStackContainer.Service.S3)
+ .withStartupTimeout(Duration.ofSeconds(30));
+
+ localStackContainer.start();
+
+ s3Client = S3Client.builder()
+ .endpointOverride(localStackContainer.getEndpoint())
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(localStackContainer.getAccessKey(), localStackContainer.getSecretKey())))
+ .region(Region.of(S3_REGION))
+ .httpClient(UrlConnectionHttpClient.builder().build())
+ .forcePathStyle(true)
+ .build();
+ }
+
+ @BeforeEach
+ public void setupRunner() {
+ runner = new StandardConnectorTestRunner.Builder()
+ .connectorClassName("org.apache.nifi.connectors.kafkas3.KafkaToS3")
+ .narLibraryDirectory(new File("target/libDir"))
+ .build();
+ assertNotNull(runner);
+ }
+
+ @AfterAll
+ public static void cleanupTestContainers() {
+ if (s3Client != null) {
+ s3Client.close();
+ }
+
+ if (localStackContainer != null) {
+ localStackContainer.stop();
+ }
+
+ if (schemaRegistryContainer != null) {
+ schemaRegistryContainer.stop();
+ }
+
+ if (kafkaContainer != null) {
+ kafkaContainer.stop();
+ }
+
+ if (network != null) {
+ network.close();
+ }
+ }
+
+ @AfterEach
+ public void cleanupRunner() throws IOException {
+ if (runner != null) {
+ runner.close();
+ }
+ }
+
+
+ private void createS3Bucket(final String bucketName) {
+ s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+ }
+
+ private void createKafkaTopics(final String... topicNames) throws ExecutionException, InterruptedException {
+ final Properties adminProps = new Properties();
+ adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9093");
+ adminProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ adminProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ adminProps.put(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";",
+ SCRAM_USERNAME, SCRAM_PASSWORD
+ ));
+
+ try (final AdminClient adminClient = AdminClient.create(adminProps)) {
+ final List<NewTopic> topics = new ArrayList<>();
+ for (final String topicName : topicNames) {
+ topics.add(new NewTopic(topicName, 1, (short) 1));
+ }
+
+ adminClient.createTopics(topics).all().get();
+ }
+ }
+
+ private void produceRecordsToTopic(final String topicName, final String... records) throws ExecutionException, InterruptedException {
+ final Properties producerProps = new Properties();
+ producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9093");
+ producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ producerProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ producerProps.put(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";",
+ SCRAM_USERNAME, SCRAM_PASSWORD
+ ));
+
+ try (final KafkaProducer<String, String> producer = new KafkaProducer<>(producerProps)) {
+ for (final String record : records) {
+ final ProducerRecord<String, String> producerRecord = new ProducerRecord<>(topicName, record);
+ producer.send(producerRecord).get();
+ }
+
+ producer.flush();
+ }
+ }
+
+ private String getSchemaRegistryUrl() {
+ return String.format("http://%s:%d", schemaRegistryContainer.getHost(), schemaRegistryContainer.getMappedPort(8081));
+ }
+
+ private void produceAvroRecordsToTopic(final String topicName, final Schema schema, final GenericRecord... records) throws ExecutionException, InterruptedException {
+ final Properties producerProps = new Properties();
+ producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9093");
+ producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
+ producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
+ producerProps.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
+ producerProps.put(SaslConfigs.SASL_JAAS_CONFIG, String.format(
+ "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";",
+ SCRAM_USERNAME, SCRAM_PASSWORD
+ ));
+ producerProps.put("schema.registry.url", getSchemaRegistryUrl());
+
+ try (final KafkaProducer<String, GenericRecord> producer = new KafkaProducer<>(producerProps)) {
+ for (final GenericRecord record : records) {
+ final ProducerRecord<String, GenericRecord> producerRecord = new ProducerRecord<>(topicName, record);
+ producer.send(producerRecord).get();
+ }
+
+ producer.flush();
+ }
+ }
+
+
+ @Test
+ public void testVerification() throws ExecutionException, InterruptedException, FlowUpdateException {
+ createKafkaTopics("topic-1", "topic-2", "topic-3", "topic-4", "topic-5", "Z-topic", "an-important-topic");
+
+ produceRecordsToTopic("topic-1",
+ """
+ {"id": 1, "name": "Alice", "age": 30}""",
+ """
+ {"id": 2, "name": "Bob", "age": 25}""",
+ """
+ {"id": 3, "name": "Charlie", "age": 35}"""
+ );
+
+ produceRecordsToTopic("an-important-topic",
+ "This is a plaintext message",
+ "Another important message",
+ "Final plaintext record"
+ );
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ runner.applyUpdate();
+
+ // Perform verification to ensure that valid server configuration passes
+ final ConnectorConfigVerificationResult connectionVerificationResults = runner.verifyConfiguration("Kafka Connection", List.of(kafkaServerConfig));
+ connectionVerificationResults.assertNoFailures();
+
+ // Apply the configuration that we've now validated
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig));
+
+ // Perform verification to ensure that valid topic configuration passes
+ final PropertyGroupConfiguration topic1Config = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "topic-1",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSuccessfulFlow",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+ final ConnectorConfigVerificationResult topic1VerificationResults = runner.verifyConfiguration("Kafka Topics", List.of(topic1Config));
+ topic1VerificationResults.assertNoFailures();
+
+ // Perform verification against a topic with invalid data for the selected data format
+ final PropertyGroupConfiguration importantTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "an-important-topic",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSuccessfulFlow",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+
+ final ConnectorConfigVerificationResult importantTopicVerificationResults = runner.verifyConfiguration("Kafka Topics", List.of(importantTopicConfig));
+ final List<ConfigVerificationResult> invalidImportantTopicResults = importantTopicVerificationResults.getFailedResults();
+ assertEquals(1, invalidImportantTopicResults.size());
+ final ConfigVerificationResult invalidResult = invalidImportantTopicResults.getFirst();
+ assertTrue(invalidResult.getExplanation().contains("parse"), "Unexpected validation reason: " + invalidResult.getExplanation());
+
+ runner.applyUpdate();
+ }
+
+ @Test
+ public void testJsonFlow() throws IOException, ExecutionException, InterruptedException, FlowUpdateException {
+ final String bucketName = "test-json-flow";
+ createS3Bucket(bucketName);
+ createKafkaTopics("story");
+
+ produceRecordsToTopic("story",
+ """
+ {"page": 1, "words": "Once upon a time, there was a NiFi developer." }""",
+ """
+ {"page": 2, "words": "The developer wanted to build a connector to move data from Kafka to S3." }""",
+ """
+ {"page": 3, "words": "After much effort, the connector was complete and worked flawlessly!" }""",
+ """
+ {"page": 4, "words": "The end." }"""
+ );
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration kafkaTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "story",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSuccessfulFlow",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+
+ final PropertyGroupConfiguration s3DestinationConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "Avro",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+ final PropertyGroupConfiguration s3CredentialsConfig = PropertyGroupConfiguration.fromStringValues("S3 Credentials", Map.of(
+ "S3 Authentication Strategy", "Access Key ID and Secret Key",
+ "S3 Access Key ID", localStackContainer.getAccessKey(),
+ "S3 Secret Access Key", localStackContainer.getSecretKey()
+ ));
+ final PropertyGroupConfiguration s3MergeConfig = PropertyGroupConfiguration.fromStringValues("Merge Configuration", Map.of(
+ "Target Object Size", "1 MB",
+ "Merge Latency", "1 sec"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig));
+ runner.configure("Kafka Topics", List.of(kafkaTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3DestinationConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ final List<ValidationResult> validationResults = runner.validate();
+ assertEquals(Collections.emptyList(), validationResults);
+
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(10));
+ runner.waitForIdle(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ verifyS3ObjectsCreated(bucketName);
+ }
+
+ private void verifyS3ObjectsCreated(final String bucketName) throws IOException {
+ final ListObjectsV2Response listResponse = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> objects = listResponse.contents();
+
+ assertFalse(objects.isEmpty(), "Expected at least one object in S3 bucket");
+
+ for (final S3Object s3Object : objects) {
+ final GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(s3Object.key()).build();
+ try (final ResponseInputStream<GetObjectResponse> objectContent = s3Client.getObject(getObjectRequest)) {
+ final long objectSize = objectContent.response().contentLength();
+ assertTrue(objectSize > 0, "Expected S3 object " + s3Object.key() + " to have content");
+ }
+ }
+ }
+
+ @Test
+ public void testSchemaRegistryVerification() throws ExecutionException, InterruptedException, FlowUpdateException {
+ createKafkaTopics("avro-topic");
+
+ final String schemaString = """
+ {
+ "type": "record",
+ "name": "TestRecord",
+ "namespace": "org.apache.nifi.test",
+ "fields": [
+ {"name": "id", "type": "int"},
+ {"name": "message", "type": "string"}
+ ]
+ }""";
+
+ final Schema schema = new Schema.Parser().parse(schemaString);
+
+ final GenericRecord record1 = new GenericData.Record(schema);
+ record1.put("id", 100);
+ record1.put("message", "Test message 1");
+
+ final GenericRecord record2 = new GenericData.Record(schema);
+ record2.put("id", 200);
+ record2.put("message", "Test message 2");
+
+ produceAvroRecordsToTopic("avro-topic", schema, record1, record2);
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration schemaRegistryConfig = PropertyGroupConfiguration.fromStringValues("Schema Registry Settings", Map.of(
+ "Schema Registry URL", getSchemaRegistryUrl()
+ ));
+
+ final ConnectorConfigVerificationResult connectionVerificationResults = runner.verifyConfiguration("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+ connectionVerificationResults.assertNoFailures();
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+
+ final PropertyGroupConfiguration avroTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "avro-topic",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSchemaRegistryVerification",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "Avro"
+ ));
+
+ final ConnectorConfigVerificationResult avroTopicVerificationResults = runner.verifyConfiguration("Kafka Topics", List.of(avroTopicConfig));
+ avroTopicVerificationResults.assertNoFailures();
+
+ runner.applyUpdate();
+ }
+
+ @Test
+ public void testWithSchemaRegistry() throws IOException, ExecutionException, InterruptedException, FlowUpdateException {
+ final String bucketName = "test-schema-registry";
+ createS3Bucket(bucketName);
+ createKafkaTopics("user-events");
+
+ final String schemaString = """
+ {
+ "type": "record",
+ "name": "UserEvent",
+ "namespace": "org.apache.nifi.test",
+ "fields": [
+ {"name": "userId", "type": "int"},
+ {"name": "userName", "type": "string"},
+ {"name": "eventType", "type": "string"},
+ {"name": "timestamp", "type": "long"}
+ ]
+ }""";
+
+ final Schema schema = new Schema.Parser().parse(schemaString);
+
+ final GenericRecord record1 = new GenericData.Record(schema);
+ record1.put("userId", 1001);
+ record1.put("userName", "alice");
+ record1.put("eventType", "login");
+ record1.put("timestamp", System.currentTimeMillis());
+
+ final GenericRecord record2 = new GenericData.Record(schema);
+ record2.put("userId", 1002);
+ record2.put("userName", "bob");
+ record2.put("eventType", "purchase");
+ record2.put("timestamp", System.currentTimeMillis());
+
+ final GenericRecord record3 = new GenericData.Record(schema);
+ record3.put("userId", 1003);
+ record3.put("userName", "charlie");
+ record3.put("eventType", "logout");
+ record3.put("timestamp", System.currentTimeMillis());
+
+ produceAvroRecordsToTopic("user-events", schema, record1, record2, record3);
+
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration schemaRegistryConfig = PropertyGroupConfiguration.fromStringValues("Schema Registry Settings", Map.of(
+ "Schema Registry URL", getSchemaRegistryUrl()
+ ));
+
+ final PropertyGroupConfiguration kafkaTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "user-events",
+ "Consumer Group ID", "nifi-kafka-to-s3-testSchemaRegistry",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "Avro"
+ ));
+
+ final PropertyGroupConfiguration s3DestinationConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "Avro",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+ final PropertyGroupConfiguration s3CredentialsConfig = PropertyGroupConfiguration.fromStringValues("S3 Credentials", Map.of(
+ "S3 Authentication Strategy", "Access Key ID and Secret Key",
+ "S3 Access Key ID", localStackContainer.getAccessKey(),
+ "S3 Secret Access Key", localStackContainer.getSecretKey()
+ ));
+ final PropertyGroupConfiguration s3MergeConfig = PropertyGroupConfiguration.fromStringValues("Merge Configuration", Map.of(
+ "Target Object Size", "1 MB",
+ "Merge Latency", "1 sec"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+ runner.configure("Kafka Topics", List.of(kafkaTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3DestinationConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ final List<ValidationResult> validationResults = runner.validate();
+ assertEquals(Collections.emptyList(), validationResults);
+
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(10));
+ runner.waitForIdle(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ verifyS3ObjectsCreated(bucketName);
+ }
+
+ @Test
+ public void testSwitchFromJsonToAvroS3Format() throws IOException, ExecutionException, InterruptedException, FlowUpdateException {
+ final String bucketName = "test-switch-formats";
+ createS3Bucket(bucketName);
+
+ // Add data to json and avro topics
+ createKafkaTopics("json", "avro");
+
+ produceRecordsToTopic("json",
+ """
+ {"id": 1, "type": "json", "data": "First JSON record"}""",
+ """
+ {"id": 2, "type": "json", "data": "Second JSON record"}""",
+ """
+ {"id": 3, "type": "json", "data": "Third JSON record"}"""
+ );
+
+ final String schemaString = """
+ {
+ "type": "record",
+ "name": "AvroRecord",
+ "namespace": "org.apache.nifi.test",
+ "fields": [
+ {"name": "id", "type": "int"},
+ {"name": "type", "type": "string"},
+ {"name": "data", "type": "string"}
+ ]
+ }""";
+
+ final Schema schema = new Schema.Parser().parse(schemaString);
+
+ final GenericRecord avroRecord1 = new GenericData.Record(schema);
+ avroRecord1.put("id", 10);
+ avroRecord1.put("type", "avro");
+ avroRecord1.put("data", "First Avro record");
+
+ final GenericRecord avroRecord2 = new GenericData.Record(schema);
+ avroRecord2.put("id", 20);
+ avroRecord2.put("type", "avro");
+ avroRecord2.put("data", "Second Avro record");
+
+ final GenericRecord avroRecord3 = new GenericData.Record(schema);
+ avroRecord3.put("id", 30);
+ avroRecord3.put("type", "avro");
+ avroRecord3.put("data", "Third Avro record");
+
+ produceAvroRecordsToTopic("avro", schema, avroRecord1, avroRecord2, avroRecord3);
+
+ // Configure Connector to consume from JSON Kafka topic and write to S3 in JSON format, but with an invalid S3 endpoint.
+ // This will cause the data to remain queued, since PutS3Object will fail to write the data.
+ final PropertyGroupConfiguration kafkaServerConfig = PropertyGroupConfiguration.fromStringValues("Kafka Server Settings", Map.of(
+ "Kafka Brokers", "localhost:9093",
+ "Security Protocol", "SASL_PLAINTEXT",
+ "SASL Mechanism", "PLAIN",
+ "Username", SCRAM_USERNAME,
+ "Password", SCRAM_PASSWORD
+ ));
+
+ final PropertyGroupConfiguration schemaRegistryConfig = PropertyGroupConfiguration.fromStringValues("Schema Registry Settings", Map.of(
+ "Schema Registry URL", getSchemaRegistryUrl()
+ ));
+
+ final PropertyGroupConfiguration jsonTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "json",
+ "Consumer Group ID", "nifi-kafka-to-s3-testReconfiguration",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "JSON"
+ ));
+
+ final PropertyGroupConfiguration s3InvalidDestinationConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "JSON",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", "http://invalid-s3-endpoint:9999"
+ ));
+ final PropertyGroupConfiguration s3CredentialsConfig = PropertyGroupConfiguration.fromStringValues("S3 Credentials", Map.of(
+ "S3 Authentication Strategy", "Access Key ID and Secret Key",
+ "S3 Access Key ID", localStackContainer.getAccessKey(),
+ "S3 Secret Access Key", localStackContainer.getSecretKey()
+ ));
+ final PropertyGroupConfiguration s3MergeConfig = PropertyGroupConfiguration.fromStringValues("Merge Configuration", Map.of(
+ "Target Object Size", "1 MB",
+ "Merge Latency", "1 sec"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig));
+ runner.configure("Kafka Topics", List.of(jsonTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3InvalidDestinationConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ // Run the Connector with the invalid S3 endpoint to queue the JSON data. Wait for data to be queued up.
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ // Apply configuration to specify the correct S3 endpoint. Keep S3 Data Format as JSON for now.
+ // This will allow the Connector to write data to S3 and properly drain the data when we switch the S3 format from JSON to Avro.
+ final PropertyGroupConfiguration s3ValidDestinationJsonConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "JSON",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+
+ runner.configure("S3 Configuration", List.of(s3ValidDestinationJsonConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ // Make sure there is no data in S3 yet.
+ final ListObjectsV2Response initialListingResponse = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> initialS3Objects = initialListingResponse.contents();
+ assertEquals(List.of(), initialS3Objects);
+
+ // Now change the S3 Data Format from JSON to Avro. This should trigger draining of the queued JSON data.
+ final PropertyGroupConfiguration s3ValidDestinationAvroConfig = PropertyGroupConfiguration.fromStringValues("S3 Destination Configuration", Map.of(
+ "S3 Region", S3_REGION,
+ "S3 Data Format", "Avro",
+ "S3 Bucket", bucketName,
+ "S3 Endpoint Override URL", localStackContainer.getEndpoint().toString()
+ ));
+
+ // Configure to consume from the Avro topic and change S3 format to Avro
+ final PropertyGroupConfiguration avroTopicConfig = PropertyGroupConfiguration.fromStringValues("Kafka Topics Configuration", Map.of(
+ "Topic Names", "avro",
+ "Consumer Group ID", "nifi-kafka-to-s3-testReconfiguration",
+ "Offset Reset", "earliest",
+ "Kafka Data Format", "Avro"
+ ));
+
+ runner.configure("Kafka Connection", List.of(kafkaServerConfig, schemaRegistryConfig));
+ runner.configure("Kafka Topics", List.of(avroTopicConfig));
+ runner.configure("S3 Configuration", List.of(s3ValidDestinationAvroConfig, s3MergeConfig, s3CredentialsConfig));
+ runner.applyUpdate();
+
+ // After draining, there should be one JSON file in S3.
+ final ListObjectsV2Response jsonOnlyListing = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> jsonObjects = jsonOnlyListing.contents();
+ assertEquals(1, jsonObjects.size());
+
+ // Start connector, wait for more data to be ingested, and then wait for the Connector to be idle.
+ runner.startConnector();
+ try {
+ runner.waitForDataIngested(Duration.ofSeconds(30));
+ runner.waitForIdle(Duration.ofSeconds(30));
+ } finally {
+ runner.stopConnector();
+ }
+
+ // Verify that there are two objects in S3: one in JSON format and one in Avro format.
+ final ListObjectsV2Response listResponse = s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build());
+ final List<S3Object> objects = listResponse.contents();
+
+ assertEquals(2, objects.size(), "Expected exactly 2 objects in S3 bucket: one for JSON data and one for Avro data");
+
+ for (final S3Object s3Object : objects) {
+ final GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(s3Object.key()).build();
+ try (final ResponseInputStream<GetObjectResponse> objectContent = s3Client.getObject(getObjectRequest)) {
+ final long objectSize = objectContent.response().contentLength();
+ assertTrue(objectSize > 0, "Expected S3 object " + s3Object.key() + " to have content");
+ }
+ }
+ }
+
+}
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/pom.xml
new file mode 100644
index 0000000000..76e6e97097
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-nar/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-to-s3-connector</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml b/nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml
new file mode 100644
index 0000000000..0fbed3c9ea
--- /dev/null
+++ b/nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connectors</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-kafka-to-s3-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-kafka-to-s3-connector</module>
+ <module>nifi-kafka-to-s3-nar</module>
+ <module>nifi-kafka-to-s3-integration-tests</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ </dependencies>
+ </dependencyManagement>
+</project>
\ No newline at end of file
diff --git a/nifi-connectors/pom.xml b/nifi-connectors/pom.xml
new file mode 100644
index 0000000000..4186b8904d
--- /dev/null
+++ b/nifi-connectors/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-connectors</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-kafka-to-s3-bundle</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-connector-utils</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
index 0fd4ebbc7a..8c95637a5e 100644
--- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
+++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
@@ -17,7 +17,9 @@
package org.apache.nifi.processors.aws.s3;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
@@ -26,6 +28,7 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.aws.AbstractAwsSyncProcessor;
+import org.apache.nifi.processors.aws.region.RegionUtil;
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.services.s3.S3Client;
@@ -34,9 +37,11 @@ import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.nifi.processors.aws.region.RegionUtil.CUSTOM_REGION;
import static org.apache.nifi.processors.aws.region.RegionUtil.REGION;
@@ -315,4 +320,11 @@ public abstract class AbstractS3Processor extends AbstractAwsSyncProcessor<S3Cli
attributes.put(S3_ENCRYPTION_STRATEGY, encryptionService.getStrategyName());
}
}
+
+ @ConnectorMethod(name="getAvailableRegions", description="Returns the list of available AWS regions")
+ public List<String> getAvailableRegions() {
+ return RegionUtil.getAwsRegionAllowableValues().stream()
+ .map(AllowableValue::getValue)
+ .toList();
+ }
}
diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java
index b5e35d8b3e..2626c41c8a 100644
--- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java
+++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-regions/src/main/java/org/apache/nifi/processors/aws/region/RegionUtil.java
@@ -100,7 +100,7 @@ public final class RegionUtil {
*
* @return All available AWS regions as AllowableValues.
*/
- private static List<AllowableValue> getAwsRegionAllowableValues() {
+ public static List<AllowableValue> getAwsRegionAllowableValues() {
final List<AllowableValue> values = new ArrayList<>();
for (final Region region : Region.regions()) {
values.add(createAwsRegionAllowableValue(region));
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
index 455c5ee5dc..a75a3ae50c 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java
@@ -26,9 +26,11 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.Validator;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.kafka.processors.common.KafkaUtils;
import org.apache.nifi.kafka.processors.consumer.OffsetTracker;
@@ -58,11 +60,14 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.VerifiableProcessor;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.RecordSetWriterFactory;
import org.apache.nifi.util.StringUtils;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
@@ -512,26 +517,110 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess
final List<ConfigVerificationResult> verificationResults = new ArrayList<>();
final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
- final PollingContext pollingContext = createPollingContext(context);
- final KafkaConsumerService consumerService = connectionService.getConsumerService(pollingContext);
+ final PollingContext pollingContext = createPollingContext(context, null, AutoOffsetReset.EARLIEST);
+ try (final KafkaConsumerService consumerService = connectionService.getConsumerService(pollingContext)) {
+ final ConfigVerificationResult partitionVerification = verifyPartitions(consumerService, pollingContext);
+ verificationResults.add(partitionVerification);
+
+ final ConfigVerificationResult parsingResult = verifyCanParse(context, consumerService, verificationLogger);
+ verificationResults.add(parsingResult);
+ } catch (final IOException e) {
+ verificationResults.add(new ConfigVerificationResult.Builder()
+ .verificationStepName("Communicate with Kafka Broker")
+ .outcome(Outcome.FAILED)
+ .explanation("There was an I/O failure when communicating with Kafka: " + e)
+ .build());
+ }
+
+ return verificationResults;
+ }
- final ConfigVerificationResult.Builder verificationPartitions = new ConfigVerificationResult.Builder()
- .verificationStepName("Verify Topic Partitions");
+ private ConfigVerificationResult verifyPartitions(final KafkaConsumerService consumerService, final PollingContext pollingContext) {
+ final ConfigVerificationResult.Builder partitionVerification = new ConfigVerificationResult.Builder()
+ .verificationStepName("Verify Topic Partitions");
try {
final List<PartitionState> partitionStates = consumerService.getPartitionStates();
- verificationPartitions
- .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
- .explanation(String.format("Partitions [%d] found for Topics %s", partitionStates.size(), pollingContext.getTopics()));
+ partitionVerification
+ .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .explanation(String.format("Found [%d] partitions for Topics %s", partitionStates.size(), pollingContext.getTopics()));
} catch (final Exception e) {
getLogger().error("Topics {} Partition verification failed", pollingContext.getTopics(), e);
- verificationPartitions
- .outcome(ConfigVerificationResult.Outcome.FAILED)
- .explanation(String.format("Topics %s Partition access failed: %s", pollingContext.getTopics(), e));
+ partitionVerification
+ .outcome(ConfigVerificationResult.Outcome.FAILED)
+ .explanation(String.format("Topics %s Partition access failed: %s", pollingContext.getTopics(), e));
}
- verificationResults.add(verificationPartitions.build());
- return verificationResults;
+ return partitionVerification.build();
+ }
+
+ private ConfigVerificationResult verifyCanParse(final ProcessContext context, final KafkaConsumerService consumerService, final ComponentLog verificationLogger) {
+ final Iterable<ByteRecord> records = consumerService.poll(Duration.ofSeconds(60));
+ final ProcessingStrategy processingStrategy = context.getProperty(PROCESSING_STRATEGY).asAllowableValue(ProcessingStrategy.class);
+ if (processingStrategy != ProcessingStrategy.RECORD) {
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.SKIPPED)
+ .explanation("Processing Strategy is set to " + processingStrategy.getValue() + " so skipping record parsing verification")
+ .build();
+ }
+
+ final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
+ int recordIndex = 0;
+ for (final ByteRecord byteRecord : records) {
+ recordIndex++;
+ final Map<String, String> recordAttributes = KafkaUtils.toAttributes(
+ byteRecord, keyEncoding, headerNamePattern, headerEncoding, commitOffsets);
+
+ try (final InputStream inputStream = new ByteArrayInputStream(byteRecord.getValue());
+ final RecordReader reader = readerFactory.createRecordReader(recordAttributes, inputStream, byteRecord.getValue().length, verificationLogger)) {
+
+ while (reader.nextRecord() != null) {
+ }
+ } catch (final Exception e) {
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.FAILED)
+ .explanation("Failed to parse Record number " + recordIndex + ": " + e)
+ .build();
+ }
+ }
+
+ if (recordIndex == 0) {
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.SKIPPED)
+ .explanation("No records were received to parse")
+ .build();
+ }
+
+ return new ConfigVerificationResult.Builder()
+ .verificationStepName("Parse Records")
+ .outcome(Outcome.SUCCESSFUL)
+ .explanation("Successfully parsed " + recordIndex + " records")
+ .build();
+ }
+
+
+ @ConnectorMethod(
+ name = "sampleTopics",
+ description = "Returns a list of sample data from the topics that would be consumed by this processor."
+ )
+ public List<byte[]> sampleTopics(final ProcessContext context) throws IOException {
+ final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class);
+ final PollingContext pollingContext = createPollingContext(context, "nifi-validation-" + System.currentTimeMillis(), AutoOffsetReset.EARLIEST);
+ try (final KafkaConsumerService consumerService = connectionService.getConsumerService(pollingContext)) {
+ final Iterable<ByteRecord> records = consumerService.poll(Duration.ofSeconds(60));
+ final List<byte[]> samples = new ArrayList<>();
+ for (final ByteRecord record : records) {
+ samples.add(record.getValue());
+ if (samples.size() >= 10) {
+ break;
+ }
+ }
+
+ return samples;
+ }
}
private KafkaConsumerService getConsumerService(final ProcessContext context) {
@@ -620,6 +709,10 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess
final String groupId = context.getProperty(GROUP_ID).getValue();
final String offsetReset = context.getProperty(AUTO_OFFSET_RESET).getValue();
final AutoOffsetReset autoOffsetReset = AutoOffsetReset.valueOf(offsetReset.toUpperCase());
+ return createPollingContext(context, groupId, autoOffsetReset);
+ }
+
+ private PollingContext createPollingContext(final ProcessContext context, final String groupId, final AutoOffsetReset autoOffsetReset) {
final String topics = context.getProperty(TOPICS).evaluateAttributeExpressions().getValue();
final String topicFormat = context.getProperty(TOPIC_FORMAT).getValue();
@@ -636,4 +729,5 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess
return pollingContext;
}
+
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java
index 531e5fff07..0fb38dabab 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java
@@ -17,6 +17,7 @@
package org.apache.nifi.kafka.processors;
import org.apache.nifi.components.ConfigVerificationResult;
+import org.apache.nifi.components.ConfigVerificationResult.Outcome;
import org.apache.nifi.kafka.service.api.KafkaConnectionService;
import org.apache.nifi.kafka.service.api.common.PartitionState;
import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
@@ -31,11 +32,13 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import static org.apache.nifi.kafka.processors.ConsumeKafka.CONNECTION_SERVICE;
import static org.apache.nifi.kafka.processors.ConsumeKafka.GROUP_ID;
import static org.apache.nifi.kafka.processors.ConsumeKafka.TOPICS;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@@ -96,10 +99,15 @@ class ConsumeKafkaTest {
runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID);
final List<ConfigVerificationResult> results = processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap());
- assertEquals(1, results.size());
+ final List<ConfigVerificationResult> successResults = results.stream()
+ .filter(result -> result.getOutcome() == ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .toList();
+ assertEquals(1, successResults.size());
- final ConfigVerificationResult firstResult = results.iterator().next();
- assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, firstResult.getOutcome());
+ final boolean anyFailures = results.stream().anyMatch(result -> result.getOutcome() == Outcome.FAILED);
+ assertFalse(anyFailures, "At least one verification result was a failure: " + results);
+
+ final ConfigVerificationResult firstResult = successResults.getFirst();
assertNotNull(firstResult.getExplanation());
}
@@ -113,9 +121,14 @@ class ConsumeKafkaTest {
runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID);
final List<ConfigVerificationResult> results = processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap());
- assertEquals(1, results.size());
+ assertEquals(2, results.size());
+
+ final List<ConfigVerificationResult> failedResults = results.stream()
+ .filter(result -> result.getOutcome() == ConfigVerificationResult.Outcome.FAILED)
+ .toList();
+ assertEquals(1, failedResults.size());
- final ConfigVerificationResult firstResult = results.iterator().next();
+ final ConfigVerificationResult firstResult = failedResults.getFirst();
assertEquals(ConfigVerificationResult.Outcome.FAILED, firstResult.getOutcome());
assertNotNull(firstResult.getExplanation());
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java
index 27b07b41e3..171ff02fdf 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/consumer/PollingContext.java
@@ -34,7 +34,7 @@ public class PollingContext {
public PollingContext(final String groupId, final Collection<String> topics,
final AutoOffsetReset autoOffsetReset) {
- this.groupId = Objects.requireNonNull(groupId, "Group ID required");
+ this.groupId = groupId;
this.topics = Collections.unmodifiableCollection(Objects.requireNonNull(topics, "Topics required"));
this.topicPattern = null;
this.autoOffsetReset = Objects.requireNonNull(autoOffsetReset, "Auto Offset Reset required");
@@ -42,7 +42,7 @@ public class PollingContext {
public PollingContext(final String groupId, final Pattern topicPattern,
final AutoOffsetReset autoOffsetReset) {
- this.groupId = Objects.requireNonNull(groupId, "Group ID required");
+ this.groupId = groupId;
this.topics = Collections.emptyList();
this.topicPattern = Objects.requireNonNull(topicPattern, "Topic Patten required");
this.autoOffsetReset = Objects.requireNonNull(autoOffsetReset, "Auto Offset Reset required");
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
index a03c0ecac3..61c71e0a03 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java
@@ -30,6 +30,8 @@ import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.ConfigVerificationResult;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
+import org.apache.nifi.components.connector.components.MethodArgument;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
@@ -42,6 +44,7 @@ import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
import org.apache.nifi.kafka.service.api.consumer.PollingContext;
import org.apache.nifi.kafka.service.api.producer.KafkaProducerService;
import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration;
+import org.apache.nifi.kafka.service.consumer.Kafka3AssignmentService;
import org.apache.nifi.kafka.service.consumer.Kafka3ConsumerService;
import org.apache.nifi.kafka.service.consumer.Subscription;
import org.apache.nifi.kafka.service.producer.Kafka3ProducerService;
@@ -64,6 +67,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -235,18 +240,25 @@ public class Kafka3ConnectionService extends AbstractControllerService implement
public KafkaConsumerService getConsumerService(final PollingContext pollingContext) {
Objects.requireNonNull(pollingContext, "Polling Context required");
- final Subscription subscription = createSubscription(pollingContext);
+ final String groupId = pollingContext.getGroupId();
final Properties properties = new Properties();
properties.putAll(consumerProperties);
- properties.put(ConsumerConfig.GROUP_ID_CONFIG, subscription.getGroupId());
- properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, subscription.getAutoOffsetReset().getValue());
+ if (groupId != null) {
+ properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
+ }
+ properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, pollingContext.getAutoOffsetReset().getValue());
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
final ByteArrayDeserializer deserializer = new ByteArrayDeserializer();
final Consumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties, deserializer, deserializer);
- return new Kafka3ConsumerService(getLogger(), consumer, subscription);
+ if (groupId == null) {
+ return new Kafka3AssignmentService(consumer, pollingContext.getTopics());
+ } else {
+ final Subscription subscription = createSubscription(pollingContext);
+ return new Kafka3ConsumerService(getLogger(), consumer, subscription);
+ }
}
private Subscription createSubscription(final PollingContext pollingContext) {
@@ -291,6 +303,26 @@ public class Kafka3ConnectionService extends AbstractControllerService implement
return uri;
}
+ @ConnectorMethod(
+ name="listTopicNames",
+ description="Returns a list of topic names available in the Kafka cluster",
+ arguments = {
+ @MethodArgument(name = "context", type = ConfigurationContext.class, description = "The configuration context that specifies connectivity details")
+ })
+ public List<String> listTopicNames(final ConfigurationContext context) throws ExecutionException, InterruptedException {
+ final Properties clientProperties = getClientProperties(context);
+ final Properties consumerProperties = getConsumerProperties(context, clientProperties);
+
+ try (final Admin admin = Admin.create(consumerProperties)) {
+ final ListTopicsResult result = admin.listTopics();
+ final Set<String> topicNames = result.names().get();
+ final List<String> sortedTopicNames = new ArrayList<>(topicNames);
+ sortedTopicNames.sort(String.CASE_INSENSITIVE_ORDER);
+ return sortedTopicNames;
+ }
+ }
+
+
@Override
public List<ConfigVerificationResult> verify(final ConfigurationContext configurationContext, final ComponentLog verificationLogger, final Map<String, String> variables) {
// Build Client Properties based on configured values and defaults from Consumer Properties
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3AssignmentService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3AssignmentService.java
new file mode 100644
index 0000000000..783134ca42
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3AssignmentService.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.kafka.service.consumer;
+
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.nifi.kafka.service.api.common.PartitionState;
+import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
+import org.apache.nifi.kafka.service.api.consumer.PollingSummary;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+public class Kafka3AssignmentService implements KafkaConsumerService {
+ private final Consumer<byte[], byte[]> consumer;
+ private volatile boolean closed = false;
+
+ public Kafka3AssignmentService(final Consumer<byte[], byte[]> consumer, final Collection<String> topicNames) {
+ this.consumer = consumer;
+
+ final List<TopicPartition> topicPartitions = new ArrayList<>();
+ for (final String topicName : topicNames) {
+ final List<PartitionInfo> partitionInfos = consumer.partitionsFor(topicName);
+ partitionInfos.forEach(info -> topicPartitions.add(new TopicPartition(info.topic(), info.partition())));
+ }
+
+ consumer.assign(topicPartitions);
+ }
+
+ @Override
+ public void commit(final PollingSummary pollingSummary) {
+ throw new UnsupportedOperationException("Commit not supported for assignment-based consumer");
+ }
+
+ @Override
+ public void rollback() {
+ // no-op
+ }
+
+ @Override
+ public boolean isClosed() {
+ return false;
+ }
+
+ @Override
+ public Iterable<ByteRecord> poll(final Duration maxWaitDuration) {
+ final ConsumerRecords<byte[], byte[]> consumerRecords = consumer.poll(maxWaitDuration);
+ if (consumerRecords.isEmpty()) {
+ return List.of();
+ }
+
+ return new RecordIterable(consumerRecords);
+ }
+
+ @Override
+ public List<PartitionState> getPartitionStates() {
+ final Set<TopicPartition> topicPartitions = consumer.assignment();
+ return topicPartitions.stream()
+ .map(tp -> new PartitionState(tp.topic(), tp.partition()))
+ .toList();
+ }
+
+ @Override
+ public void close() {
+ if (closed) {
+ return;
+ }
+
+ closed = true;
+ consumer.close();
+ }
+}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
index 59ea23818a..b1152fb249 100644
--- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/Kafka3ConsumerService.java
@@ -18,7 +18,6 @@ package org.apache.nifi.kafka.service.consumer;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
@@ -27,13 +26,11 @@ import org.apache.nifi.kafka.service.api.common.PartitionState;
import org.apache.nifi.kafka.service.api.common.TopicPartitionSummary;
import org.apache.nifi.kafka.service.api.consumer.KafkaConsumerService;
import org.apache.nifi.kafka.service.api.consumer.PollingSummary;
-import org.apache.nifi.kafka.service.api.header.RecordHeader;
import org.apache.nifi.kafka.service.api.record.ByteRecord;
import org.apache.nifi.logging.ComponentLog;
import java.io.Closeable;
import java.time.Duration;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -179,58 +176,4 @@ public class Kafka3ConsumerService implements KafkaConsumerService, Closeable, C
return offsets;
}
-
-
- private static class RecordIterable implements Iterable<ByteRecord> {
- private final Iterator<ByteRecord> records;
-
- private RecordIterable(final Iterable<ConsumerRecord<byte[], byte[]>> consumerRecords) {
- this.records = new RecordIterator(consumerRecords);
- }
-
- @Override
- public Iterator<ByteRecord> iterator() {
- return records;
- }
- }
-
- private static class RecordIterator implements Iterator<ByteRecord> {
- private final Iterator<ConsumerRecord<byte[], byte[]>> consumerRecords;
-
- private RecordIterator(final Iterable<ConsumerRecord<byte[], byte[]>> records) {
- this.consumerRecords = records.iterator();
- }
-
- @Override
- public boolean hasNext() {
- return consumerRecords.hasNext();
- }
-
- @Override
- public ByteRecord next() {
- final ConsumerRecord<byte[], byte[]> consumerRecord = consumerRecords.next();
- final List<RecordHeader> recordHeaders = new ArrayList<>();
- consumerRecord.headers().forEach(header -> {
- final RecordHeader recordHeader = new RecordHeader(header.key(), header.value());
- recordHeaders.add(recordHeader);
- });
-
- // Support Kafka tombstones
- byte[] value = consumerRecord.value();
- if (value == null) {
- value = new byte[0];
- }
-
- return new ByteRecord(
- consumerRecord.topic(),
- consumerRecord.partition(),
- consumerRecord.offset(),
- consumerRecord.timestamp(),
- recordHeaders,
- consumerRecord.key(),
- value,
- 1
- );
- }
- }
}
diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/RecordIterable.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/RecordIterable.java
new file mode 100644
index 0000000000..714caa06db
--- /dev/null
+++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/consumer/RecordIterable.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.kafka.service.consumer;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.nifi.kafka.service.api.header.RecordHeader;
+import org.apache.nifi.kafka.service.api.record.ByteRecord;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class RecordIterable implements Iterable<ByteRecord> {
+ private final Iterator<ByteRecord> records;
+
+ public RecordIterable(final Iterable<ConsumerRecord<byte[], byte[]>> consumerRecords) {
+ this.records = new RecordIterator(consumerRecords);
+ }
+
+ @Override
+ public Iterator<ByteRecord> iterator() {
+ return records;
+ }
+
+ private static class RecordIterator implements Iterator<ByteRecord> {
+ private final Iterator<ConsumerRecord<byte[], byte[]>> consumerRecords;
+
+ private RecordIterator(final Iterable<ConsumerRecord<byte[], byte[]>> records) {
+ this.consumerRecords = records.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return consumerRecords.hasNext();
+ }
+
+ @Override
+ public ByteRecord next() {
+ final ConsumerRecord<byte[], byte[]> consumerRecord = consumerRecords.next();
+ final List<RecordHeader> recordHeaders = new ArrayList<>();
+ consumerRecord.headers().forEach(header -> {
+ final RecordHeader recordHeader = new RecordHeader(header.key(), header.value());
+ recordHeaders.add(recordHeader);
+ });
+
+ // Support Kafka tombstones
+ byte[] value = consumerRecord.value();
+ if (value == null) {
+ value = new byte[0];
+ }
+
+ return new ByteRecord(
+ consumerRecord.topic(),
+ consumerRecord.partition(),
+ consumerRecord.offset(),
+ consumerRecord.timestamp(),
+ recordHeaders,
+ consumerRecord.key(),
+ value,
+ 1
+ );
+ }
+ }
+}
\ No newline at end of file
diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java
index 3297dc6217..94f8b0ac1d 100644
--- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java
+++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/DuplicateFlowFile.java
@@ -43,7 +43,7 @@ import java.util.Set;
+ "generated copies are sent to the 'success' relationship. In addition, each FlowFile gets an attribute 'copy.index' set to the copy number, where the original FlowFile gets "
+ "a value of zero, and all copies receive incremented integer values.")
@WritesAttributes({
- @WritesAttribute(attribute = "copy.index", description = "A zero-based incrementing integer value based on which copy the FlowFile is.")
+ @WritesAttribute(attribute = "copy.index", description = "A zero-based incrementing integer value based on which copy the FlowFile is.")
})
public class DuplicateFlowFile extends AbstractProcessor {
diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
index 737fbe3884..3233465e3c 100644
--- a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
+++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
@@ -17,6 +17,8 @@
package org.apache.nifi.controller.queue;
+import java.util.concurrent.Future;
+
/**
* Represents the status of a Drop FlowFile Request that has been issued to
* a {@link FlowFileQueue}. When a queue is requested to drop its FlowFiles,
@@ -75,4 +77,9 @@ public interface DropFlowFileStatus {
* @return the reason that the state is set to a Failure state, or <code>null</code> if the state is not {@link DropFlowFileState#FAILURE}.
*/
String getFailureReason();
+
+ /**
+ * @return a Future that can be used to determine when the drop operation has completed
+ */
+ Future<Void> getCompletionFuture();
}
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
index 8f59bd721c..9be7a17f52 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
@@ -48,7 +48,7 @@ public class TestVolatileProvenanceRepository {
@Test
public void testAddAndGet() throws IOException {
- repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));
+ repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties((String) null));
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
@@ -81,7 +81,7 @@ public class TestVolatileProvenanceRepository {
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
- repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));
+ repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties((String) null));
final String uuid = "00000000-0000-0000-0000-000000000000";
final Map<String, String> attributes = new HashMap<>();
@@ -120,7 +120,7 @@ public class TestVolatileProvenanceRepository {
@Test
public void testSearchForInverseValue() throws InterruptedException {
- repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));
+ repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties((String) null));
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java b/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java
index 0856573c9d..33a29d880b 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java
@@ -44,7 +44,8 @@ public enum ResourceType {
Operation("/operation"),
Tenant("/tenants"),
ParameterContext("/parameter-contexts"),
- ParameterProvider("/parameter-providers");
+ ParameterProvider("/parameter-providers"),
+ Connector("/connectors");
final String value;
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java
new file mode 100644
index 0000000000..780d496a14
--- /dev/null
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ClusteredConnectorRequestReplicator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.cluster.coordination.http.replication;
+
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+import jakarta.ws.rs.core.Response.Status.Family;
+import jakarta.ws.rs.core.Response.StatusType;
+import org.apache.nifi.components.connector.ConnectorRequestReplicator;
+import org.apache.nifi.components.connector.ConnectorState;
+import org.apache.nifi.controller.flow.FlowManager;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class ClusteredConnectorRequestReplicator implements ConnectorRequestReplicator {
+ private static final String GET = "GET";
+
+ private final Supplier<RequestReplicator> requestReplicatorSupplier;
+
+ public ClusteredConnectorRequestReplicator(final Supplier<RequestReplicator> requestReplicatorSupplier) {
+ this.requestReplicatorSupplier = Objects.requireNonNull(requestReplicatorSupplier, "Request Replicator Supplier required");
+ }
+
+ @Override
+ public ConnectorState getState(final String connectorId) throws IOException {
+ final RequestReplicator requestReplicator = getRequestReplicator();
+ final AsyncClusterResponse asyncResponse = requestReplicator.replicate(GET, URI.create("/nifi-api/connectors/" + connectorId + "/status"), null, Map.of());
+ try {
+ final Response response = asyncResponse.awaitMergedResponse().getClientResponse();
+ verifyResponse(response.getStatusInfo(), connectorId);
+
+ // TODO: Need a ConnectorStatusEntity? Need correct URI.
+ return response.readEntity(ConnectorState.class);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while waiting for response for status of Connector with ID " + connectorId, e);
+ }
+ }
+
+ @Override
+ public void setFlowManager(final FlowManager flowManager) {
+ }
+
+ private RequestReplicator getRequestReplicator() {
+ final RequestReplicator requestReplicator = requestReplicatorSupplier.get();
+ return Objects.requireNonNull(requestReplicator, "Request Replicator required");
+ }
+
+ private void verifyResponse(final StatusType responseStatusType, final String connectorId) throws IOException {
+ final int statusCode = responseStatusType.getStatusCode();
+ final String reason = responseStatusType.getReasonPhrase();
+
+ if (responseStatusType.getStatusCode() == Status.NOT_FOUND.getStatusCode()) {
+ throw new IllegalArgumentException("Connector with ID + " + connectorId + " does not exist");
+ }
+
+ final Family responseFamily = responseStatusType.getFamily();
+ if (responseFamily == Family.SERVER_ERROR) {
+ throw new IOException("Server-side error requesting State for Connector with ID + " + connectorId + ". Status code: " + statusCode + ", reason: " + reason);
+ }
+ if (responseFamily == Family.CLIENT_ERROR) {
+ throw new IOException("Client-side error requesting State for Connector with ID " + connectorId + ". Status code: " + statusCode + ", reason: " + reason);
+ }
+ if (responseFamily != Family.SUCCESSFUL) {
+ throw new IOException("Unexpected response code " + statusCode + " returned when requesting State for Connector with ID " + connectorId + ": " + reason);
+ }
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/ConnectorRequestReplicatorConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/ConnectorRequestReplicatorConfiguration.java
new file mode 100644
index 0000000000..4024321bd1
--- /dev/null
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/ConnectorRequestReplicatorConfiguration.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.framework.configuration;
+
+import org.apache.nifi.cluster.coordination.http.replication.ClusteredConnectorRequestReplicator;
+import org.apache.nifi.cluster.coordination.http.replication.RequestReplicator;
+import org.apache.nifi.components.connector.ConnectorRequestReplicator;
+import org.apache.nifi.components.connector.StandaloneConnectorRequestReplicator;
+import org.apache.nifi.util.NiFiProperties;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ConnectorRequestReplicatorConfiguration {
+
+ @Bean
+ public ConnectorRequestReplicator connectorRequestReplicator(
+ final NiFiProperties nifiProperties,
+ final ObjectProvider<RequestReplicator> requestReplicatorProvider) {
+
+ if (nifiProperties.isClustered()) {
+ // We have to use an ObjectProvider here and obtain a Supplier because of a circular dependency.
+ // The request replicator will not be available when created. However, it will be
+ // available before attempting to use the ConnectorRequestReplicator.
+ return new ClusteredConnectorRequestReplicator(requestReplicatorProvider::getIfAvailable);
+ }
+
+ return new StandaloneConnectorRequestReplicator();
+ }
+}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java
index 1bdbde5093..b0eb1edd9d 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/framework/configuration/FrameworkClusterConfiguration.java
@@ -48,8 +48,6 @@ public class FrameworkClusterConfiguration {
private EventReporter eventReporter;
- private FlowController flowController;
-
private ClusterCoordinator clusterCoordinator;
private WebClientService webClientService;
@@ -59,11 +57,6 @@ public class FrameworkClusterConfiguration {
this.properties = properties;
}
- @Autowired
- public void setFlowController(final FlowController flowController) {
- this.flowController = flowController;
- }
-
@Autowired
public void setEventReporter(final EventReporter eventReporter) {
this.eventReporter = eventReporter;
@@ -106,7 +99,7 @@ public class FrameworkClusterConfiguration {
}
@Bean
- public ClusterDecommissionTask decommissionTask() {
+ public ClusterDecommissionTask decommissionTask(final FlowController flowController) {
return new ClusterDecommissionTask(clusterCoordinator, flowController);
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
index a21cd535c1..9bdde6d3d9 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
@@ -268,7 +268,7 @@ public class TestThreadPoolRequestReplicator {
when(coordinator.getConnectionStatus(Mockito.any(NodeIdentifier.class))).thenReturn(new NodeConnectionStatus(nodeId, NodeConnectionState.CONNECTED));
final AtomicInteger requestCount = new AtomicInteger(0);
- final NiFiProperties props = NiFiProperties.createBasicNiFiProperties(null);
+ final NiFiProperties props = NiFiProperties.createBasicNiFiProperties((String) null);
final MockReplicationClient client = new MockReplicationClient();
final RequestCompletionCallback requestCompletionCallback = (uri, method, responses) -> {
@@ -329,7 +329,7 @@ public class TestThreadPoolRequestReplicator {
final ClusterCoordinator coordinator = createClusterCoordinator();
final AtomicInteger requestCount = new AtomicInteger(0);
- final NiFiProperties props = NiFiProperties.createBasicNiFiProperties(null);
+ final NiFiProperties props = NiFiProperties.createBasicNiFiProperties((String) null);
final MockReplicationClient client = new MockReplicationClient();
final RequestCompletionCallback requestCompletionCallback = (uri, method, responses) -> {
@@ -529,7 +529,7 @@ public class TestThreadPoolRequestReplicator {
private void withReplicator(final WithReplicator function, final Status status, final long delayMillis, final RuntimeException failure,
final String expectedRequestChain, final String expectedProxiedEntityGroups) {
final ClusterCoordinator coordinator = createClusterCoordinator();
- final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties(null);
+ final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties((String) null);
final MockReplicationClient client = new MockReplicationClient();
final RequestCompletionCallback requestCompletionCallback = (uri, method, responses) -> {
};
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
index 20f5363744..b7e431aa27 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java
@@ -42,12 +42,18 @@ import org.apache.nifi.components.ConfigurableComponent;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.InvocationFailedException;
+import org.apache.nifi.components.connector.components.ComponentState;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
+import org.apache.nifi.components.connector.components.MethodArgument;
import org.apache.nifi.components.validation.ValidationState;
import org.apache.nifi.components.validation.ValidationStatus;
import org.apache.nifi.components.validation.ValidationTrigger;
import org.apache.nifi.connectable.Connectable;
+import org.apache.nifi.connectable.ConnectableFlowFileActivity;
import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.connectable.FlowFileActivity;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.controller.exception.ProcessorInstantiationException;
import org.apache.nifi.controller.scheduling.LifecycleState;
@@ -178,6 +184,8 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
private volatile BackoffMechanism backoffMechanism;
private volatile String maxBackoffPeriod;
+ private final ConnectableFlowFileActivity flowFileActivity = new ConnectableFlowFileActivity();
+
public StandardProcessorNode(final LoggableComponent<Processor> processor, final String uuid,
final ValidationContextFactory validationContextFactory, final ProcessScheduler scheduler,
final ControllerServiceProvider controllerServiceProvider, final ReloadComponent reloadComponent,
@@ -977,7 +985,9 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
}
@Override
- public List<ConfigVerificationResult> verifyConfiguration(final ProcessContext context, final ComponentLog logger, final Map<String, String> attributes, final ExtensionManager extensionManager) {
+ public List<ConfigVerificationResult> verifyConfiguration(final ProcessContext context, final ComponentLog logger, final Map<String, String> attributes, final ExtensionManager extensionManager,
+ final ParameterLookup parameterLookup) {
+
final List<ConfigVerificationResult> results = new ArrayList<>();
try {
@@ -985,7 +995,7 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
final long startNanos = System.nanoTime();
// Call super's verifyConfig, which will perform component validation
- results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), getProcessGroup().getParameterContext()));
+ results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), parameterLookup));
final long validationComplete = System.nanoTime();
// If any invalid outcomes from validation, we do not want to perform additional verification, because we only run additional verification when the component is valid.
@@ -1628,6 +1638,9 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
// Completion Timestamp is set to MAX_VALUE because we don't want to timeout until the task has a chance to run.
final AtomicLong completionTimestampRef = new AtomicLong(Long.MAX_VALUE);
+ // Mark current time as latest activity time so that we don't show as idle when the processor was stopped.
+ flowFileActivity.reset();
+
// Create a task to invoke the @OnScheduled annotation of the processor
final Callable<Void> startupTask = () -> {
final ScheduledState currentScheduleState = scheduledState.get();
@@ -1922,6 +1935,86 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
}
}
+ @Override
+ public List<ConnectorMethod> getConnectorMethods() {
+ return getConnectorMethods(getProcessor().getClass());
+ }
+
+ public Object invokeConnectorMethod(final String methodName, final Map<String, Object> arguments, final ProcessContext processContext) throws InvocationFailedException {
+ final ConfigurableComponent component = getComponent();
+
+ try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(getExtensionManager(), component.getClass(), getIdentifier())) {
+ final Method implementationMethod = discoverConnectorMethod(component.getClass(), methodName);
+ final MethodArgument[] methodArguments = getConnectorMethodArguments(methodName, implementationMethod, component);
+ final List<Object> argumentValues = new ArrayList<>();
+ for (final MethodArgument methodArgument : methodArguments) {
+ final Object argumentValue = arguments.get(methodArgument.name());
+ if (ProcessContext.class.equals(methodArgument.type())) {
+ continue;
+ }
+
+ if (argumentValue == null && methodArgument.required()) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the required argument '"
+ + methodArgument.name() + "' was not provided");
+ }
+
+ if (argumentValue != null && !(methodArgument.type().isAssignableFrom(argumentValue.getClass()))) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the argument '"
+ + methodArgument.name() + "' is of type " + argumentValue.getClass().getName() + " defined by " + argumentValue.getClass().getClassLoader()
+ + " but the method expects type " + methodArgument.type().getName() + " defined by " + methodArgument.type().getClassLoader());
+ }
+
+ argumentValues.add(argumentValue);
+ }
+
+ // Inject ProcessContext if the method signature supports it
+ final Class<?>[] argumentTypes = implementationMethod.getParameterTypes();
+ if (argumentTypes.length > 0 && ProcessContext.class.isAssignableFrom(argumentTypes[0])) {
+ argumentValues.addFirst(processContext);
+ }
+ if (argumentTypes.length > 1 && ProcessContext.class.isAssignableFrom(argumentTypes[argumentTypes.length - 1])) {
+ argumentValues.add(processContext);
+ }
+
+ try {
+ implementationMethod.setAccessible(true);
+ return implementationMethod.invoke(component, argumentValues.toArray());
+ } catch (final Exception e) {
+ throw new InvocationFailedException(e);
+ }
+ }
+ }
+
+ private MethodArgument[] getConnectorMethodArguments(final String methodName, final Method implementationMethod, final ConfigurableComponent component) throws InvocationFailedException {
+ if (implementationMethod == null) {
+ throw new InvocationFailedException("No such connector method '" + methodName + "' exists for " + component.getClass().getName());
+ }
+
+ final ConnectorMethod connectorMethodDefinition = implementationMethod.getAnnotation(ConnectorMethod.class);
+ final ComponentState[] componentStates = connectorMethodDefinition.allowedStates();
+ final ComponentState currentState = getComponentState();
+ final boolean validState = Set.of(componentStates).contains(currentState);
+ if (!validState) {
+ throw new IllegalStateException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because Processor is in state " + currentState
+ + " but the Connector Method does not allow invocation in this state");
+ }
+
+ final MethodArgument[] methodArguments = connectorMethodDefinition.arguments();
+ return methodArguments;
+ }
+
+ private ComponentState getComponentState() {
+ final ScheduledState scheduledState = getScheduledState();
+
+ return switch (scheduledState) {
+ case DISABLED -> ComponentState.PROCESSOR_DISABLED;
+ case STOPPED -> ComponentState.STOPPED;
+ case RUNNING, RUN_ONCE -> ComponentState.RUNNING;
+ case STARTING -> ComponentState.STARTING;
+ case STOPPING -> ComponentState.STOPPING;
+ };
+ }
+
/**
* Marks the processor as fully stopped, and completes any futures that are to be completed as a result
*/
@@ -2167,4 +2260,9 @@ public class StandardProcessorNode extends ProcessorNode implements Connectable
serviceNode.updateReference(this, descriptor);
}
}
+
+ @Override
+ public FlowFileActivity getFlowFileActivity() {
+ return flowFileActivity;
+ }
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java
index 50308465d2..a08000428c 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java
@@ -637,7 +637,7 @@ public abstract class AbstractFlowManager implements FlowManager {
throw new IllegalStateException("Cannot create Parameter Context with name '" + name + "' because a Parameter Context already exists with that name");
}
- final ParameterReferenceManager referenceManager = new StandardParameterReferenceManager(this);
+ final ParameterReferenceManager referenceManager = new StandardParameterReferenceManager(this::getRootGroup);
final ParameterContext parameterContext = new StandardParameterContext.Builder()
.id(id)
.name(name)
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java
index 7c8e8963b8..6e6c58bd92 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java
@@ -19,13 +19,13 @@ package org.apache.nifi.controller.flowanalysis;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.registry.flow.mapping.ComponentIdLookup;
import org.apache.nifi.registry.flow.mapping.FlowMappingOptions;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.registry.flow.mapping.VersionedComponentStateLookup;
public class FlowAnalysisUtil {
public static final String ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE = "*****";
- public static NiFiRegistryFlowMapper createMapper(ExtensionManager extensionManager) {
+ public static VersionedComponentFlowMapper createMapper(ExtensionManager extensionManager) {
final FlowMappingOptions flowMappingOptions = new FlowMappingOptions.Builder()
.mapPropertyDescriptors(true)
.mapControllerServiceReferencesToVersionedId(true)
@@ -36,7 +36,7 @@ public class FlowAnalysisUtil {
.mapAssetReferences(true)
.build();
- final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(extensionManager, flowMappingOptions) {
+ final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(extensionManager, flowMappingOptions) {
@Override
public String getGroupId(String groupId) {
return groupId;
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index a12f7a626e..f8a1014f39 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -596,7 +596,10 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
// Update the FlowFile Repository
try {
final Collection<StandardRepositoryRecord> repoRecords = checkpoint.records.values();
- context.getFlowFileRepository().updateRepository((Collection) repoRecords);
+ if (!repoRecords.isEmpty()) {
+ context.getFlowFileRepository().updateRepository((Collection) repoRecords);
+ context.getConnectable().getFlowFileActivity().updateLatestActivityTime();
+ }
} catch (final IOException ioe) {
// if we fail to commit the session, we need to roll back
// the checkpoints as well because none of the checkpoints
@@ -974,7 +977,7 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
// Therefore, we create an Iterable that can iterate over each of these events, modifying them as needed, and returning them
// in the appropriate order. This prevents an unnecessary step of creating an intermediate List and adding all of those values
// to the List.
- // This is done in a similar veign to how Java 8's streams work, iterating over the events and returning a processed version
+ // This is done in a similar vein to how Java 8's streams work, iterating over the events and returning a processed version
// one-at-a-time as opposed to iterating over the entire Collection and putting the results in another Collection. However,
// we don't want to change the Framework to require Java 8 at this time, because it's not yet as prevalent as we would desire
final Map<String, FlowFileRecord> flowFileRecordMap = new HashMap<>();
@@ -983,6 +986,9 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
flowFileRecordMap.put(flowFile.getAttribute(CoreAttributes.UUID.key()), flowFile);
}
+ // Update connectable to indicate how much data was received and sent.
+ updateTransferCounts(recordsToSubmit);
+
final long commitNanos = System.nanoTime();
final List<ProvenanceEventRecord> autoTermEvents = checkpoint.autoTerminatedEvents;
final Iterable<ProvenanceEventRecord> iterable = new Iterable<>() {
@@ -1029,6 +1035,23 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
provenanceRepo.registerEvents(iterable);
}
+ private void updateTransferCounts(final Collection<ProvenanceEventRecord> events) {
+ int createdCount = 0;
+ long createdBytes = 0L;
+ for (final ProvenanceEventRecord event : events) {
+ if (event.getEventType() == ProvenanceEventType.CREATE) {
+ createdCount++;
+ createdBytes += event.getFileSize();
+ }
+ }
+
+ context.getConnectable().getFlowFileActivity().updateTransferCounts(
+ provenanceReporter.getFlowFilesReceived() + createdCount,
+ // For bytes, consider received and fetched; we don't include fetched in count, as the FlowFile count didn't change.
+ provenanceReporter.getBytesReceived() + provenanceReporter.getBytesFetched() + createdBytes,
+ provenanceReporter.getFlowFilesSent(),
+ provenanceReporter.getBytesSent());
+ }
private void updateEventContentClaims(final ProvenanceEventBuilder builder, final FlowFile flowFile, final StandardRepositoryRecord repoRecord) {
final ContentClaim originalClaim = repoRecord.getOriginalClaim();
@@ -1237,6 +1260,10 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
resetWriteClaims();
resetReadClaim();
+ if (!recordValues.isEmpty()) {
+ context.getConnectable().getFlowFileActivity().updateLatestActivityTime();
+ }
+
if (recordsToHandle.isEmpty()) {
LOG.trace("{} was rolled back, but no events were performed by this ProcessSession", this);
acknowledgeRecords();
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java
index 9b87040540..3cb82157ca 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceInvocationHandler.java
@@ -17,6 +17,8 @@
package org.apache.nifi.controller.service;
import org.apache.commons.lang3.ClassUtils;
+import org.apache.nifi.components.connector.components.ComponentState;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.ControllerServiceProxyWrapper;
import org.apache.nifi.nar.ExtensionManager;
@@ -29,7 +31,6 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -38,15 +39,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class StandardControllerServiceInvocationHandler implements ControllerServiceInvocationHandler {
private static final Logger logger = LoggerFactory.getLogger(StandardControllerServiceInvocationHandler.class);
private static final Method PROXY_WRAPPER_GET_WRAPPED_METHOD;
-
- private static final Set<Method> validDisabledMethods;
static {
- // methods that are okay to be called when the service is disabled.
- final Set<Method> validMethods = new HashSet<>();
- validMethods.addAll(Arrays.asList(ControllerService.class.getMethods()));
- validMethods.addAll(Arrays.asList(Object.class.getMethods()));
- validDisabledMethods = Collections.unmodifiableSet(validMethods);
-
try {
PROXY_WRAPPER_GET_WRAPPED_METHOD = ControllerServiceProxyWrapper.class.getMethod("getWrapped");
} catch (final NoSuchMethodException e) {
@@ -54,6 +47,7 @@ public class StandardControllerServiceInvocationHandler implements ControllerSer
}
}
+ private final Set<Method> validDisabledMethods;
private final ControllerService originalService;
private final AtomicReference<ControllerServiceNode> serviceNodeHolder = new AtomicReference<>(null);
private final ExtensionManager extensionManager;
@@ -73,6 +67,34 @@ public class StandardControllerServiceInvocationHandler implements ControllerSer
this.extensionManager = extensionManager;
this.originalService = originalService;
this.serviceNodeHolder.set(serviceNode);
+ validDisabledMethods = determineDisabledAllowedMethods(originalService);
+ }
+
+ private Set<Method> determineDisabledAllowedMethods(final ControllerService implementation) {
+ // methods that are okay to be called when the service is disabled.
+ final Set<Method> validMethods = new HashSet<>();
+ validMethods.addAll(Arrays.asList(ControllerService.class.getMethods()));
+ validMethods.addAll(Arrays.asList(Object.class.getMethods()));
+
+ // If there are any Connector Methods that are allowed to be called when the service is STOPPED or STOPPING, add those as well.
+ // This allows us to perform actions from Connectors, such as obtaining a list of available resources, during Connector configuration, etc.
+ // while the service is currently stopped.
+ for (final Method method : implementation.getClass().getDeclaredMethods()) {
+ final ConnectorMethod connectorMethod = method.getAnnotation(ConnectorMethod.class);
+ if (connectorMethod == null) {
+ continue;
+ }
+
+ final ComponentState[] allowedStates = connectorMethod.allowedStates();
+ for (final ComponentState allowedState : allowedStates) {
+ if (allowedState == ComponentState.STOPPED || allowedState == ComponentState.STOPPING) {
+ validMethods.add(method);
+ break;
+ }
+ }
+ }
+
+ return validMethods;
}
@Override
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
index a530513cf0..25c6f03da9 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
@@ -35,6 +35,10 @@ import org.apache.nifi.components.ConfigurableComponent;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.connector.InvocationFailedException;
+import org.apache.nifi.components.connector.components.ComponentState;
+import org.apache.nifi.components.connector.components.ConnectorMethod;
+import org.apache.nifi.components.connector.components.MethodArgument;
import org.apache.nifi.components.validation.ValidationState;
import org.apache.nifi.components.validation.ValidationStatus;
import org.apache.nifi.components.validation.ValidationTrigger;
@@ -95,6 +99,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
public class StandardControllerServiceNode extends AbstractComponentNode implements ControllerServiceNode {
@@ -504,7 +509,7 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
@Override
public List<ConfigVerificationResult> verifyConfiguration(final ConfigurationContext context, final ComponentLog logger, final Map<String, String> variables,
- final ExtensionManager extensionManager) {
+ final ExtensionManager extensionManager, final ParameterLookup parameterLookup) {
final List<ConfigVerificationResult> results = new ArrayList<>();
@@ -513,7 +518,7 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
final long startNanos = System.nanoTime();
// Call super's verifyConfig, which will perform component validation
- results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), getProcessGroup() == null ? null : getProcessGroup().getParameterContext()));
+ results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), parameterLookup));
final long validationComplete = System.nanoTime();
// If any invalid outcomes from validation, we do not want to perform additional verification, because we only run additional verification when the component is valid.
@@ -673,6 +678,7 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
LOG.debug("Validation rescheduling rejected for {}", serviceNode, e);
future.completeExceptionally(new IllegalStateException("Enabling %s rejected: Validation Status [%s] Errors %s".formatted(serviceNode, validationStatus, errors)));
}
+
// Enable command rescheduled or rejected
return;
}
@@ -899,6 +905,88 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme
}
}
+ // TODO: Refactor, this is the same between ProcessorNode / ControllerServiceNode except for getComponentState
+ @Override
+ public Object invokeConnectorMethod(final String methodName, final Map<String, Object> arguments, final ConfigurationContext configurationContext) throws InvocationFailedException {
+ final ConfigurableComponent component = getComponent();
+
+ try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(getExtensionManager(), component.getClass(), getIdentifier())) {
+ final Method implementationMethod = discoverConnectorMethod(component.getClass(), methodName);
+ final MethodArgument[] methodArguments = getConnectorMethodArguments(methodName, implementationMethod, component);
+ final List<Object> argumentValues = new ArrayList<>();
+ for (final MethodArgument methodArgument : methodArguments) {
+ final Object argumentValue = arguments.get(methodArgument.name());
+ if (ConfigurationContext.class.equals(methodArgument.type())) {
+ continue;
+ }
+
+ if (argumentValue == null && methodArgument.required()) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the required argument '"
+ + methodArgument.name() + "' was not provided");
+ }
+
+ if (argumentValue != null && !(methodArgument.type().isAssignableFrom(argumentValue.getClass()))) {
+ throw new IllegalArgumentException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because the argument '"
+ + methodArgument.name() + "' is of type " + argumentValue.getClass().getName() + " defined by " + argumentValue.getClass().getClassLoader()
+ + " but the method expects type " + methodArgument.type().getName() + " defined by " + methodArgument.type().getClassLoader());
+ }
+
+ argumentValues.add(argumentValue);
+ }
+
+ // Inject ConfigurationContext if the method signature supports it
+ // TODO: Can we move away from Maps and instead just use reflection to get the arguments directly?
+ final Class<?>[] argumentTypes = implementationMethod.getParameterTypes();
+ if (argumentTypes.length > 0 && ConfigurationContext.class.isAssignableFrom(argumentTypes[0])) {
+ argumentValues.addFirst(configurationContext);
+ }
+ if (argumentTypes.length > 1 && ConfigurationContext.class.isAssignableFrom(argumentTypes[argumentTypes.length - 1])) {
+ argumentValues.add(configurationContext);
+ }
+
+ try {
+ implementationMethod.setAccessible(true);
+ return implementationMethod.invoke(component, argumentValues.toArray());
+ } catch (final Exception e) {
+ throw new InvocationFailedException(e);
+ }
+ }
+ }
+
+ @Override
+ public List<ConnectorMethod> getConnectorMethods() {
+ return getConnectorMethods(getControllerServiceImplementation().getClass());
+ }
+
+ private MethodArgument[] getConnectorMethodArguments(final String methodName, final Method implementationMethod, final ConfigurableComponent component) throws InvocationFailedException {
+ if (implementationMethod == null) {
+ throw new InvocationFailedException("No such connector method '" + methodName + "' exists for " + component.getClass().getName());
+ }
+
+ final ConnectorMethod connectorMethodDefinition = implementationMethod.getAnnotation(ConnectorMethod.class);
+ final ComponentState[] componentStates = connectorMethodDefinition.allowedStates();
+ final ComponentState currentState = getComponentState();
+ final boolean validState = Set.of(componentStates).contains(currentState);
+ if (!validState) {
+ throw new IllegalStateException("Cannot invoke Connector Method '" + methodName + "' on " + this + " because Processor is in state " + currentState
+ + " but the Connector Method does not allow invocation in this state");
+ }
+
+ final MethodArgument[] methodArguments = connectorMethodDefinition.arguments();
+ return methodArguments;
+ }
+
+ private ComponentState getComponentState() {
+ final ControllerServiceState scheduledState = getState();
+
+ return switch (scheduledState) {
+ case DISABLED -> ComponentState.STOPPED;
+ case DISABLING -> ComponentState.STOPPING;
+ case ENABLING -> ComponentState.STARTING;
+ case ENABLED -> ComponentState.RUNNING;
+ };
+ }
+
@Override
protected void performFlowAnalysisOnThis() {
getValidationContextFactory().getFlowAnalyzer().ifPresent(flowAnalyzer -> flowAnalyzer.analyzeControllerService(this));
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
index f1945c8ed5..95b3f34afa 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
@@ -320,7 +320,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi
}
@Override
- public Future<Void> enableControllerServicesAsync(final Collection<ControllerServiceNode> serviceNodes) {
+ public CompletableFuture<Void> enableControllerServicesAsync(final Collection<ControllerServiceNode> serviceNodes) {
final CompletableFuture<Void> future = new CompletableFuture<>();
processScheduler.submitFrameworkTask(() -> {
try {
@@ -392,7 +392,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi
}
@Override
- public Future<Void> enableControllerServiceAndDependencies(final ControllerServiceNode serviceNode) {
+ public CompletableFuture<Void> enableControllerServiceAndDependencies(final ControllerServiceNode serviceNode) {
if (serviceNode.isActive()) {
logger.debug("Enabling of Controller Service {} triggered but service already enabled", serviceNode);
return CompletableFuture.completedFuture(null);
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java
index 7b165b8a6c..442d301fdd 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceResolver.java
@@ -30,7 +30,7 @@ import org.apache.nifi.flow.VersionedPropertyDescriptor;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.registry.flow.FlowSnapshotContainer;
import org.apache.nifi.registry.flow.RegisteredFlowSnapshot;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import java.util.ArrayList;
import java.util.Collection;
@@ -47,13 +47,13 @@ public class StandardControllerServiceResolver implements ControllerServiceResol
private final Authorizer authorizer;
private final FlowManager flowManager;
- private final NiFiRegistryFlowMapper flowMapper;
+ private final VersionedComponentFlowMapper flowMapper;
private final ControllerServiceProvider controllerServiceProvider;
private final ControllerServiceApiLookup controllerServiceApiLookup;
public StandardControllerServiceResolver(final Authorizer authorizer,
final FlowManager flowManager,
- final NiFiRegistryFlowMapper flowMapper,
+ final VersionedComponentFlowMapper flowMapper,
final ControllerServiceProvider controllerServiceProvider,
final ControllerServiceApiLookup controllerServiceApiLookup) {
this.authorizer = authorizer;
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java
index 7ea59751e8..3779f5b344 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java
@@ -290,6 +290,7 @@ public class VolatileBulletinRepository implements BulletinRepository {
case FLOW_ANALYSIS_RULE -> FLOW_ANALYSIS_RULE_BULLETIN_STORE_KEY;
case PARAMETER_PROVIDER -> PARAMETER_PROVIDER_BULLETIN_STORE_KEY;
case FLOW_REGISTRY_CLIENT -> FLOW_REGISTRY_CLIENT_STORE_KEY;
+ case CONNECTOR -> bulletin.getSourceId();
default -> bulletin.getGroupId();
};
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
index 0d9775fadd..3466d317e4 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
@@ -112,7 +112,7 @@ import org.apache.nifi.registry.flow.diff.FlowDifference;
import org.apache.nifi.registry.flow.diff.StandardComparableDataFlow;
import org.apache.nifi.registry.flow.diff.StandardFlowComparator;
import org.apache.nifi.registry.flow.diff.StaticDifferenceDescriptor;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.RemoteGroupPort;
import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor;
@@ -311,7 +311,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
@Override
public void synchronize(final ProcessGroup group, final VersionedExternalFlow versionedExternalFlow, final FlowSynchronizationOptions options) {
- final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(context.getExtensionManager(), context.getFlowMappingOptions());
+ final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(context.getExtensionManager(), context.getFlowMappingOptions());
final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(group, context.getControllerServiceProvider(), context.getFlowManager(), true);
final ComparableDataFlow localFlow = new StandardComparableDataFlow("Currently Loaded Flow", versionedGroup);
@@ -998,7 +998,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
private <T extends Connectable> Map<String, T> componentsById(final ProcessGroup group, final Function<ProcessGroup, Collection<T>> retrieveComponents) {
return retrieveComponents.apply(group).stream()
.collect(Collectors.toMap(component -> component.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier())), Function.identity()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier())), Function.identity()));
}
private <T> Map<String, T> componentsById(final ProcessGroup group, final Function<ProcessGroup, Collection<T>> retrieveComponents,
@@ -1006,7 +1006,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
return retrieveComponents.apply(group).stream()
.collect(Collectors.toMap(component -> retrieveVersionedComponentId.apply(component).orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(retrieveId.apply(component))), Function.identity()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(retrieveId.apply(component))), Function.identity()));
}
private void synchronizeFunnels(final ProcessGroup group, final VersionedProcessGroup proposed, final Map<String, Funnel> funnelsByVersionedId) {
@@ -1242,7 +1242,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final Map<String, Port> removedInputPortsByVersionId = new HashMap<>();
group.getInputPorts()
.forEach(port -> removedInputPortsByVersionId.put(port.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
+ VersionedComponentFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
flowContents.getInputPorts().stream()
.map(VersionedPort::getIdentifier)
@@ -1261,7 +1261,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final Map<String, Port> removedOutputPortsByVersionId = new HashMap<>();
group.getOutputPorts()
.forEach(port -> removedOutputPortsByVersionId.put(port.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
+ VersionedComponentFlowMapper.generateVersionedComponentId(port.getIdentifier())), port));
flowContents.getOutputPorts().stream()
.map(VersionedPort::getIdentifier)
@@ -1288,7 +1288,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
group.findAllProcessors()
.forEach(proc -> proposedProcessors.remove(proc.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(proc.getIdentifier()))));
+ VersionedComponentFlowMapper.generateVersionedComponentId(proc.getIdentifier()))));
for (final VersionedProcessor processorToAdd : proposedProcessors.values()) {
final String processorToAddClass = processorToAdd.getType();
@@ -1319,7 +1319,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
group.findAllControllerServices()
.forEach(service -> proposedServices.remove(service.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(service.getIdentifier()))));
+ VersionedComponentFlowMapper.generateVersionedComponentId(service.getIdentifier()))));
for (final VersionedControllerService serviceToAdd : proposedServices.values()) {
final String serviceToAddClass = serviceToAdd.getType();
@@ -1348,7 +1348,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
group.findAllConnections()
.forEach(conn -> proposedConnections.remove(conn.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(conn.getIdentifier()))));
+ VersionedComponentFlowMapper.generateVersionedComponentId(conn.getIdentifier()))));
for (final VersionedConnection connectionToAdd : proposedConnections.values()) {
if (connectionToAdd.getPrioritizers() != null) {
@@ -3030,7 +3030,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
private String getServiceInstanceId(final String serviceVersionedComponentId, final ProcessGroup group) {
for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) {
final String versionedId = serviceNode.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
if (versionedId.equals(serviceVersionedComponentId)) {
return serviceNode.getIdentifier();
}
@@ -3650,7 +3650,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final String rpgId = connectableComponent.getGroupId();
final Optional<RemoteProcessGroup> rpgOption = group.getRemoteProcessGroups().stream()
.filter(component -> rpgId.equals(component.getIdentifier()) || rpgId.equals(component.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
+ VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
.findAny();
if (rpgOption.isEmpty()) {
@@ -3676,7 +3676,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
final String rpgId = connectableComponent.getGroupId();
final Optional<RemoteProcessGroup> rpgOption = group.getRemoteProcessGroups().stream()
.filter(component -> rpgId.equals(component.getIdentifier()) || rpgId.equals(component.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
+ VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier()))))
.findAny();
if (rpgOption.isEmpty()) {
@@ -3877,12 +3877,12 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
}
private <T extends org.apache.nifi.components.VersionedComponent & Connectable> boolean matchesId(final T component, final String id) {
- return id.equals(component.getIdentifier()) || id.equals(component.getVersionedComponentId().orElse(NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier())));
+ return id.equals(component.getIdentifier()) || id.equals(component.getVersionedComponentId().orElse(VersionedComponentFlowMapper.generateVersionedComponentId(component.getIdentifier())));
}
private boolean matchesGroupId(final ProcessGroup group, final String groupId) {
return groupId.equals(group.getIdentifier()) || group.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(group.getIdentifier())).equals(groupId);
+ VersionedComponentFlowMapper.generateVersionedComponentId(group.getIdentifier())).equals(groupId);
}
private void findAllProcessors(final Set<VersionedProcessor> processors, final Set<VersionedProcessGroup> childGroups, final Map<String, VersionedProcessor> map) {
@@ -3934,7 +3934,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
// match group's current connections to proposed connections to determine if they've been removed
for (final Connection connection : processGroup.getConnections()) {
final String versionedId = connection.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(connection.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(connection.getIdentifier()));
final VersionedConnection proposedConnection = proposedConnectionsByVersionedId.get(versionedId);
if (proposedConnection == null) {
// connection doesn't exist in proposed connections, make sure it doesn't have any data in it
@@ -3953,7 +3953,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
// match current child groups to proposed child groups to determine if they've been removed
for (final ProcessGroup childGroup : processGroup.getProcessGroups()) {
final String versionedId = childGroup.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(childGroup.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(childGroup.getIdentifier()));
final VersionedProcessGroup proposedChildGroup = proposedGroupsByVersionedId.get(versionedId);
if (proposedChildGroup == null) {
if (verifyConnectionRemoval) {
@@ -3979,7 +3979,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) {
final String serviceNodeVersionedComponentId = serviceNode.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
+ VersionedComponentFlowMapper.generateVersionedComponentId(serviceNode.getIdentifier()));
if (serviceNodeVersionedComponentId.equals(versionedComponentId)) {
return serviceNode;
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
index bc8d717645..9fbfde6f0a 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
@@ -34,11 +34,13 @@ import org.apache.nifi.components.validation.ValidationStatus;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.connectable.FlowFileActivity;
import org.apache.nifi.connectable.Funnel;
import org.apache.nifi.connectable.LocalPort;
import org.apache.nifi.connectable.Port;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.connectable.Positionable;
+import org.apache.nifi.connectable.ProcessGroupFlowFileActivity;
import org.apache.nifi.controller.ComponentNode;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerService;
@@ -105,7 +107,7 @@ import org.apache.nifi.registry.flow.diff.StandardComparableDataFlow;
import org.apache.nifi.registry.flow.diff.StandardFlowComparator;
import org.apache.nifi.registry.flow.mapping.ComponentIdLookup;
import org.apache.nifi.registry.flow.mapping.FlowMappingOptions;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.registry.flow.mapping.VersionedComponentStateLookup;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.RemoteGroupPort;
@@ -138,6 +140,7 @@ import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -196,6 +199,8 @@ public final class StandardProcessGroup implements ProcessGroup {
private volatile ExecutionEngine executionEngine = ExecutionEngine.INHERITED;
private volatile int maxConcurrentTasks = 1;
private volatile String statelessFlowTimeout = "1 min";
+ private volatile Authorizable explicitParentAuthorizable;
+ private final FlowFileActivity flowFileActivity = new ProcessGroupFlowFileActivity(this);
private FlowFileConcurrency flowFileConcurrency = FlowFileConcurrency.UNBOUNDED;
private volatile FlowFileGate flowFileGate = new UnboundedFlowFileGate();
@@ -295,7 +300,7 @@ public final class StandardProcessGroup implements ProcessGroup {
@Override
public Authorizable getParentAuthorizable() {
- return getParent();
+ return explicitParentAuthorizable == null ? getParent() : explicitParentAuthorizable;
}
@Override
@@ -1720,7 +1725,7 @@ public final class StandardProcessGroup implements ProcessGroup {
}
@Override
- public Future<Void> startProcessor(final ProcessorNode processor, final boolean failIfStopping) {
+ public CompletableFuture<Void> startProcessor(final ProcessorNode processor, final boolean failIfStopping) {
readLock.lock();
try {
if (getProcessor(processor.getIdentifier()) == null) {
@@ -3931,7 +3936,7 @@ public final class StandardProcessGroup implements ProcessGroup {
// because this allows us to find the Controller Service when doing a Flow Diff.
ancestorServiceIds = parentGroup.getControllerServices(true).stream()
.map(cs -> cs.getVersionedComponentId().orElse(
- NiFiRegistryFlowMapper.generateVersionedComponentId(cs.getIdentifier())))
+ VersionedComponentFlowMapper.generateVersionedComponentId(cs.getIdentifier())))
.collect(Collectors.toSet());
}
@@ -3976,7 +3981,7 @@ public final class StandardProcessGroup implements ProcessGroup {
}
try {
- final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(extensionManager);
+ final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(extensionManager);
final VersionedProcessGroup versionedGroup = mapper.mapProcessGroup(this, controllerServiceProvider, flowManager, false);
final ComparableDataFlow currentFlow = new StandardComparableDataFlow("Local Flow", versionedGroup);
@@ -4551,6 +4556,48 @@ public final class StandardProcessGroup implements ProcessGroup {
return statelessFlowTimeout;
}
+ @Override
+ public FlowFileActivity getFlowFileActivity() {
+ return flowFileActivity;
+ }
+
+ @Override
+ public void setExplicitParentAuthorizable(final Authorizable parent) {
+ this.explicitParentAuthorizable = parent;
+ }
+
+ @Override
+ public CompletableFuture<Void> purge() {
+ final CompletableFuture<Void> purgeFuture = new CompletableFuture<>();
+
+ final Thread purgeThread = Thread.ofVirtual().name("Purge " + this).start(() -> {
+ try {
+ stopProcessing().get();
+ controllerServiceProvider.disableControllerServicesAsync(getControllerServices(true)).get();
+ purgeQueues();
+ removeComponents(this);
+
+ purgeFuture.complete(null);
+ } catch (final Throwable t) {
+ purgeFuture.completeExceptionally(t);
+ }
+ });
+
+ return purgeFuture;
+ }
+
+ private void purgeQueues() throws ExecutionException, InterruptedException {
+ for (final Connection connection : getConnections()) {
+ final FlowFileQueue flowFileQueue = connection.getFlowFileQueue();
+ if (flowFileQueue.isEmpty()) {
+ continue;
+ }
+
+ final DropFlowFileStatus status = connection.getFlowFileQueue().dropFlowFiles("purge-queues-" + getIdentifier(), "Framework");
+ status.getCompletionFuture().get();
+ }
+ }
+
@Override
public void setStatelessFlowTimeout(final String statelessFlowTimeout) {
if (statelessFlowTimeout == null) {
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ConnectorLogObserver.java
similarity index 72%
copy from nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
copy to nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ConnectorLogObserver.java
index 7209492ca3..d6eba91f98 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ConnectorLogObserver.java
@@ -14,26 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.nifi.logging;
-import org.apache.nifi.controller.ProcessorNode;
+import org.apache.nifi.components.connector.ConnectorNode;
import org.apache.nifi.events.BulletinFactory;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.Severity;
-/**
- * Observes processor log messages and converts them into Bulletins.
- */
-public class ProcessorLogObserver implements LogObserver {
-
+public class ConnectorLogObserver implements LogObserver {
private static final String CATEGORY = "Log Message";
private final BulletinRepository bulletinRepository;
- private final ProcessorNode processorNode;
+ private final ConnectorNode connectorNode;
- public ProcessorLogObserver(BulletinRepository bulletinRepository, ProcessorNode processorNode) {
+ public ConnectorLogObserver(final BulletinRepository bulletinRepository, final ConnectorNode connectorNode) {
this.bulletinRepository = bulletinRepository;
- this.processorNode = processorNode;
+ this.connectorNode = connectorNode;
}
@Override
@@ -41,14 +38,12 @@ public class ProcessorLogObserver implements LogObserver {
// Map LogLevel.WARN to Severity.WARNING so that we are consistent with the Severity enumeration. Else, just use whatever
// the LogLevel is (INFO and ERROR map directly and all others we will just accept as they are).
final String bulletinLevel = (message.getLogLevel() == LogLevel.WARN) ? Severity.WARNING.name() : message.getLogLevel().toString();
- bulletinRepository.addBulletin(
- BulletinFactory.createBulletin(processorNode, CATEGORY, bulletinLevel, message.getMessage(), message.getFlowFileUuid(), message.getThrowable())
- );
+ bulletinRepository.addBulletin(BulletinFactory.createBulletin(connectorNode, CATEGORY, bulletinLevel, message.getMessage()));
}
@Override
public String getComponentDescription() {
- return processorNode.toString();
+ return connectorNode.toString();
}
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
index 7209492ca3..597b3c8c10 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/ProcessorLogObserver.java
@@ -31,7 +31,7 @@ public class ProcessorLogObserver implements LogObserver {
private final BulletinRepository bulletinRepository;
private final ProcessorNode processorNode;
- public ProcessorLogObserver(BulletinRepository bulletinRepository, ProcessorNode processorNode) {
+ public ProcessorLogObserver(final BulletinRepository bulletinRepository, final ProcessorNode processorNode) {
this.bulletinRepository = bulletinRepository;
this.processorNode = processorNode;
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java
index a05d5c57ab..1944f12751 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java
@@ -22,7 +22,6 @@ import org.apache.nifi.controller.ComponentNode;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.PropertyConfiguration;
-import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.groups.ProcessGroup;
@@ -33,14 +32,15 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
+import java.util.function.Supplier;
public class StandardParameterReferenceManager implements ParameterReferenceManager {
- private final FlowManager flowManager;
+ private final Supplier<ProcessGroup> rootGroupLookup;
// TODO: Consider reworking this so that we don't have to recurse through all components all the time and instead
// have a 'caching' impl that AbstractComponentNode.setProperties() adds to/subtracts from.
- public StandardParameterReferenceManager(final FlowManager flowManager) {
- this.flowManager = flowManager;
+ public StandardParameterReferenceManager(final Supplier<ProcessGroup> rootGroupLookup) {
+ this.rootGroupLookup = rootGroupLookup;
}
@Override
@@ -57,14 +57,14 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana
public List<ParameterReferencedControllerServiceData> getReferencedControllerServiceData(final ParameterContext parameterContext, final String parameterName) {
final List<ParameterReferencedControllerServiceData> referencedControllerServiceData = new ArrayList<>();
+ final ProcessGroup rootGroup = rootGroupLookup.get();
final String versionedServiceId = parameterContext.getParameter(parameterName)
.map(Parameter::getValue)
- .map(this.flowManager::getControllerServiceNode)
+ .map(serviceId -> rootGroup.findControllerService(serviceId, true, true))
.flatMap(VersionedComponent::getVersionedComponentId)
.orElse(null);
- final ProcessGroup rootGroup = flowManager.getRootGroup();
final List<ProcessGroup> referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext));
for (final ProcessGroup group : referencingGroups) {
@@ -90,7 +90,7 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana
@Override
public Set<ProcessGroup> getProcessGroupsBound(final ParameterContext parameterContext) {
- final ProcessGroup rootGroup = flowManager.getRootGroup();
+ final ProcessGroup rootGroup = rootGroupLookup.get();
final List<ProcessGroup> referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext));
return new HashSet<>(referencingGroups);
@@ -100,14 +100,13 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana
final Function<ProcessGroup, Collection<T>> componentFunction) {
final Set<T> referencingComponents = new HashSet<>();
- final ProcessGroup rootGroup = flowManager.getRootGroup();
+ final ProcessGroup rootGroup = rootGroupLookup.get();
final List<ProcessGroup> referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext));
for (final ProcessGroup group : referencingGroups) {
for (final T componentNode : componentFunction.apply(group)) {
if (componentNode.isReferencingParameter(parameterName)) {
referencingComponents.add(componentNode);
- continue;
}
}
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java
index 5edcb7c026..f82b8eb24b 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContext.java
@@ -37,7 +37,7 @@ import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.expression.ExpressionLanguageCompiler;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.parameter.Parameter;
-import org.apache.nifi.parameter.ParameterContext;
+import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.parameter.ParameterReference;
import java.util.Collection;
@@ -59,7 +59,7 @@ public class StandardValidationContext implements ValidationContext {
private final String annotationData;
private final String groupId;
private final String componentId;
- private final ParameterContext parameterContext;
+ private final ParameterLookup parameterLookup;
private final AtomicReference<Map<PropertyDescriptor, String>> effectiveValuesRef = new AtomicReference<>();
private final boolean validateConnections;
@@ -69,7 +69,7 @@ public class StandardValidationContext implements ValidationContext {
final String annotationData,
final String groupId,
final String componentId,
- final ParameterContext parameterContext,
+ final ParameterLookup parameterLookup,
final boolean validateConnections) {
this.controllerServiceProvider = controllerServiceProvider;
@@ -77,7 +77,7 @@ public class StandardValidationContext implements ValidationContext {
this.annotationData = annotationData;
this.groupId = groupId;
this.componentId = componentId;
- this.parameterContext = parameterContext;
+ this.parameterLookup = parameterLookup;
this.validateConnections = validateConnections;
preparedQueries = new HashMap<>(properties.size());
@@ -85,7 +85,7 @@ public class StandardValidationContext implements ValidationContext {
final PropertyDescriptor desc = entry.getKey();
final PropertyConfiguration configuration = entry.getValue();
- String value = (configuration == null) ? null : configuration.getEffectiveValue(parameterContext);
+ String value = (configuration == null) ? null : configuration.getEffectiveValue(parameterLookup);
if (value == null) {
value = desc.getDefaultValue();
}
@@ -103,12 +103,12 @@ public class StandardValidationContext implements ValidationContext {
@Override
public PropertyValue newPropertyValue(final String rawValue) {
final ResourceContext resourceContext = new StandardResourceContext(new StandardResourceReferenceFactory(), null);
- return new StandardPropertyValue(resourceContext, rawValue, controllerServiceProvider, parameterContext, Query.prepareWithParametersPreEvaluated(rawValue));
+ return new StandardPropertyValue(resourceContext, rawValue, controllerServiceProvider, parameterLookup, Query.prepareWithParametersPreEvaluated(rawValue));
}
@Override
public ExpressionLanguageCompiler newExpressionLanguageCompiler() {
- return new StandardExpressionLanguageCompiler(parameterContext);
+ return new StandardExpressionLanguageCompiler(parameterLookup);
}
@Override
@@ -123,9 +123,9 @@ public class StandardValidationContext implements ValidationContext {
@Override
public PropertyValue getProperty(final PropertyDescriptor property) {
final PropertyConfiguration configuredValue = properties.get(property);
- final String effectiveValue = configuredValue == null ? property.getDefaultValue() : configuredValue.getEffectiveValue(parameterContext);
+ final String effectiveValue = configuredValue == null ? property.getDefaultValue() : configuredValue.getEffectiveValue(parameterLookup);
final ResourceContext resourceContext = new StandardResourceContext(new StandardResourceReferenceFactory(), property);
- return new StandardPropertyValue(resourceContext, effectiveValue, controllerServiceProvider, parameterContext, preparedQueries.get(property));
+ return new StandardPropertyValue(resourceContext, effectiveValue, controllerServiceProvider, parameterLookup, preparedQueries.get(property));
}
@Override
@@ -139,7 +139,7 @@ public class StandardValidationContext implements ValidationContext {
for (final Map.Entry<PropertyDescriptor, PropertyConfiguration> entry : this.properties.entrySet()) {
final PropertyDescriptor descriptor = entry.getKey();
final PropertyConfiguration configuration = entry.getValue();
- final String value = configuration == null ? descriptor.getDefaultValue() : configuration.getEffectiveValue(parameterContext);
+ final String value = configuration == null ? descriptor.getDefaultValue() : configuration.getEffectiveValue(parameterLookup);
valueMap.put(entry.getKey(), value);
}
@@ -215,20 +215,20 @@ public class StandardValidationContext implements ValidationContext {
@Override
public boolean isParameterDefined(final String parameterName) {
- if (parameterContext == null) {
+ if (parameterLookup == null) {
return false;
}
- return parameterContext.getParameter(parameterName).isPresent();
+ return parameterLookup.getParameter(parameterName).isPresent();
}
@Override
public boolean isParameterSet(final String parameterName) {
- if (parameterContext == null) {
+ if (parameterLookup == null) {
return false;
}
- final Optional<Parameter> parameterOption = parameterContext.getParameter(parameterName);
+ final Optional<Parameter> parameterOption = parameterLookup.getParameter(parameterName);
if (!parameterOption.isPresent()) {
return false;
}
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java
index 00a7b6852a..a1d8178f91 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java
@@ -22,7 +22,7 @@ import org.apache.nifi.controller.PropertyConfiguration;
import org.apache.nifi.controller.ValidationContextFactory;
import org.apache.nifi.controller.flowanalysis.FlowAnalyzer;
import org.apache.nifi.controller.service.ControllerServiceProvider;
-import org.apache.nifi.parameter.ParameterContext;
+import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.validation.RuleViolationsManager;
import java.util.Map;
@@ -50,8 +50,8 @@ public class StandardValidationContextFactory implements ValidationContextFactor
@Override
public ValidationContext newValidationContext(final Map<PropertyDescriptor, PropertyConfiguration> properties, final String annotationData, final String groupId, final String componentId,
- final ParameterContext parameterContext, final boolean validateConnections) {
- return new StandardValidationContext(serviceProvider, properties, annotationData, groupId, componentId, parameterContext, validateConnections);
+ final ParameterLookup parameterLookup, final boolean validateConnections) {
+ return new StandardValidationContext(serviceProvider, properties, annotationData, groupId, componentId, parameterLookup, validateConnections);
}
@Override
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/FlowAnalyzingRegistryClientNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/FlowAnalyzingRegistryClientNode.java
index 2fb9cae6fe..14d555d013 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/FlowAnalyzingRegistryClientNode.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/FlowAnalyzingRegistryClientNode.java
@@ -44,7 +44,7 @@ import org.apache.nifi.parameter.ParameterContext;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.parameter.ParameterUpdate;
import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedProcessGroup;
-import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
+import org.apache.nifi.registry.flow.mapping.VersionedComponentFlowMapper;
import org.apache.nifi.validation.RuleViolation;
import org.apache.nifi.validation.RuleViolationsManager;
import org.slf4j.Logger;
@@ -69,7 +69,7 @@ public final class FlowAnalyzingRegistryClientNode implements FlowRegistryClient
... 14802 lines suppressed ...