You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2023/10/14 20:05:40 UTC
[nifi] branch main updated: NIFI-12205: Moved loading of Python dependencies into background thread when processor created instead of during startup. Some code cleanup.
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new cbdf32ab79 NIFI-12205: Moved loading of Python dependencies into background thread when processor created instead of during startup. Some code cleanup.
cbdf32ab79 is described below
commit cbdf32ab79fe2041a167bee9abf69f68d91a3be6
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Mon Oct 9 16:26:27 2023 -0400
NIFI-12205: Moved loading of Python dependencies into background thread when processor created instead of during startup. Some code cleanup.
This closes #7863
Signed-off-by: David Handermann <ex...@apache.org>
---
.../nifi/components/AsyncLoadedProcessor.java | 38 +++++++
.../nifi/controller/AbstractComponentNode.java | 59 ++++++++---
.../nifi-py4j-bundle/nifi-py4j-bridge/pom.xml | 4 +
.../nifi/py4j/ProcessorCreationWorkflow.java | 28 +++++
.../org/apache/nifi/py4j/StandardPythonBridge.java | 20 +++-
.../nifi/py4j/StandardPythonProcessorBridge.java | 114 ++++++++++++++-------
.../python/processor/FlowFileTransformProxy.java | 22 ++--
.../python/processor/PythonProcessorProxy.java | 93 +++++++++++------
.../python/processor/RecordTransformProxy.java | 110 ++++++--------------
.../PythonControllerInteractionIT.java | 31 ++++--
.../src/main/python/src/nifiapi/properties.py | 20 ++--
.../nifi-python-framework-api/pom.xml | 4 +
.../org/apache/nifi/python/PythonController.java | 18 ++++
.../apache/nifi/python/PythonProcessorDetails.java | 5 +
.../python/processor/PythonProcessorBridge.java | 18 +++-
.../src/main/python/framework/Controller.py | 22 ++--
.../src/main/python/framework/ExtensionManager.py | 112 ++++++++++++--------
.../tests/system/python/PythonProcessorIT.java | 2 +-
.../test/resources/conf/pythonic/nifi.properties | 2 +
19 files changed, 473 insertions(+), 249 deletions(-)
diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/components/AsyncLoadedProcessor.java b/nifi-framework-api/src/main/java/org/apache/nifi/components/AsyncLoadedProcessor.java
new file mode 100644
index 0000000000..14f7e31cb8
--- /dev/null
+++ b/nifi-framework-api/src/main/java/org/apache/nifi/components/AsyncLoadedProcessor.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.components;
+
+public interface AsyncLoadedProcessor {
+ default boolean isLoaded() {
+ return getState() == LoadState.FINISHED_LOADING;
+ }
+
+ LoadState getState();
+
+ enum LoadState {
+ DOWNLOADING_DEPENDENCIES,
+
+ LOADING_PROCESSOR_CODE,
+
+ DEPENDENCY_DOWNLOAD_FAILED,
+
+ LOADING_PROCESSOR_CODE_FAILED,
+
+ FINISHED_LOADING
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java
index e0f24739b3..31631c6173 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java
@@ -23,6 +23,7 @@ import org.apache.nifi.attribute.expression.language.VariableImpact;
import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.bundle.BundleCoordinate;
import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.AsyncLoadedProcessor;
import org.apache.nifi.components.ClassloaderIsolationKeyProvider;
import org.apache.nifi.components.ConfigVerificationResult;
import org.apache.nifi.components.ConfigVerificationResult.Outcome;
@@ -397,16 +398,7 @@ public abstract class AbstractComponentNode implements ComponentNode {
try {
final long startNanos = System.nanoTime();
- final Map<PropertyDescriptor, PropertyConfiguration> descriptorToConfigMap = new LinkedHashMap<>();
- for (final Map.Entry<PropertyDescriptor, String> entry : propertyValues.entrySet()) {
- final PropertyDescriptor descriptor = entry.getKey();
- final String rawValue = entry.getValue();
- final String propertyValue = rawValue == null ? descriptor.getDefaultValue() : rawValue;
-
- final PropertyConfiguration propertyConfiguration = new PropertyConfiguration(propertyValue, null, Collections.emptyList(), VariableImpact.NEVER_IMPACTED);
- descriptorToConfigMap.put(descriptor, propertyConfiguration);
- }
-
+ final Map<PropertyDescriptor, PropertyConfiguration> descriptorToConfigMap = createDescriptorToConfigMap(propertyValues);
final ValidationContext validationContext = getValidationContextFactory().newValidationContext(descriptorToConfigMap, annotationData,
getProcessGroupIdentifier(), getIdentifier(), parameterContext, false);
@@ -457,6 +449,19 @@ public abstract class AbstractComponentNode implements ComponentNode {
return results;
}
+ private static Map<PropertyDescriptor, PropertyConfiguration> createDescriptorToConfigMap(final Map<PropertyDescriptor, String> propertyValues) {
+ final Map<PropertyDescriptor, PropertyConfiguration> descriptorToConfigMap = new LinkedHashMap<>();
+ for (final Map.Entry<PropertyDescriptor, String> entry : propertyValues.entrySet()) {
+ final PropertyDescriptor descriptor = entry.getKey();
+ final String rawValue = entry.getValue();
+ final String propertyValue = rawValue == null ? descriptor.getDefaultValue() : rawValue;
+
+ final PropertyConfiguration propertyConfiguration = new PropertyConfiguration(propertyValue, null, Collections.emptyList(), VariableImpact.NEVER_IMPACTED);
+ descriptorToConfigMap.put(descriptor, propertyConfiguration);
+ }
+ return descriptorToConfigMap;
+ }
+
@Override
public Set<String> getReferencedParameterNames() {
return Collections.unmodifiableSet(parameterReferenceCounts.keySet());
@@ -842,6 +847,25 @@ public abstract class AbstractComponentNode implements ComponentNode {
);
}
+ final ConfigurableComponent component = getComponent();
+ if (component instanceof final AsyncLoadedProcessor asyncLoadedProcessor) {
+ if (!asyncLoadedProcessor.isLoaded()) {
+ final String explanation = switch (asyncLoadedProcessor.getState()) {
+ case DEPENDENCY_DOWNLOAD_FAILED -> "Failed to download one or more Processor dependencies. See logs for additional details.";
+ case DOWNLOADING_DEPENDENCIES -> "In the process of downloading third-party dependencies required by the Processor.";
+ case LOADING_PROCESSOR_CODE -> "In the process of loading Processor code";
+ case LOADING_PROCESSOR_CODE_FAILED -> "Failed to parse or load Processor code. See logs for additional details.";
+ default -> null;
+ };
+
+ return Collections.singletonList(new ValidationResult.Builder()
+ .subject("Processor")
+ .explanation(explanation)
+ .valid(false)
+ .build());
+ }
+ }
+
final List<ValidationResult> invalidParameterResults = validateParameterReferences(validationContext);
invalidParameterResults.addAll(validateConfig());
@@ -851,9 +875,8 @@ public abstract class AbstractComponentNode implements ComponentNode {
return invalidParameterResults;
}
- final List<ValidationResult> validationResults = new ArrayList<>();
final Collection<ValidationResult> results = getComponent().validate(validationContext);
- validationResults.addAll(results);
+ final List<ValidationResult> validationResults = new ArrayList<>(results);
// validate selected controller services implement the API required by the processor
final Collection<ValidationResult> referencedServiceValidationResults = validateReferencedControllerServices(validationContext);
@@ -1423,10 +1446,18 @@ public abstract class AbstractComponentNode implements ComponentNode {
return context;
}
+ boolean cacheValidationContext = true;
+ if (getComponent() instanceof final AsyncLoadedProcessor asyncLoadedProcessor) {
+ cacheValidationContext = asyncLoadedProcessor.isLoaded();
+ }
+
context = getValidationContextFactory().newValidationContext(getProperties(), getAnnotationData(), getProcessGroupIdentifier(), getIdentifier(), getParameterContext(), true);
- this.validationContext = context;
- logger.debug("Updating validation context to {}", context);
+ if (cacheValidationContext) {
+ this.validationContext = context;
+ logger.debug("Updating validation context to {}", context);
+ }
+
return context;
} finally {
lock.unlock();
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml
index d24d2854aa..b2b9123f55 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml
@@ -35,6 +35,10 @@
<artifactId>nifi-python-framework-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-api</artifactId>
+ </dependency>
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/ProcessorCreationWorkflow.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/ProcessorCreationWorkflow.java
new file mode 100644
index 0000000000..7b99f146f5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/ProcessorCreationWorkflow.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.py4j;
+
+import org.apache.nifi.python.processor.PythonProcessorAdapter;
+
+public interface ProcessorCreationWorkflow {
+
+ void downloadDependencies();
+
+ PythonProcessorAdapter createProcessor();
+
+}
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
index 54bea88f6b..faa1afdd2d 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java
@@ -99,12 +99,24 @@ public class StandardPythonBridge implements PythonBridge {
final String workDirPath = processConfig.getPythonWorkingDirectory().getAbsolutePath();
final PythonController controller = pythonProcess.getController();
- final PythonProcessorAdapter processorAdapter = controller.createProcessor(type, version, workDirPath);
+
+ final ProcessorCreationWorkflow creationWorkflow = new ProcessorCreationWorkflow() {
+ @Override
+ public void downloadDependencies() {
+ controller.downloadDependencies(type, version, workDirPath);
+ }
+
+ @Override
+ public PythonProcessorAdapter createProcessor() {
+ return controller.createProcessor(type, version, workDirPath);
+ }
+ };
+
+ final PythonProcessorDetails processorDetails = controller.getProcessorDetails(type, version);
final PythonProcessorBridge processorBridge = new StandardPythonProcessorBridge.Builder()
.controller(controller)
- .processorAdapter(processorAdapter)
- .processorType(type)
- .processorVersion(version)
+ .creationWorkflow(creationWorkflow)
+ .processorDetails(processorDetails)
.workingDirectory(processConfig.getPythonWorkingDirectory())
.moduleFile(new File(controller.getModuleFile(type, version)))
.build();
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java
index 89bb85eb3f..7ae9cef081 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java
@@ -19,37 +19,44 @@ package org.apache.nifi.py4j;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.python.PythonController;
+import org.apache.nifi.python.PythonProcessorDetails;
import org.apache.nifi.python.processor.FlowFileTransform;
import org.apache.nifi.python.processor.FlowFileTransformProxy;
-import org.apache.nifi.python.processor.PythonProcessor;
import org.apache.nifi.python.processor.PythonProcessorAdapter;
import org.apache.nifi.python.processor.PythonProcessorBridge;
import org.apache.nifi.python.processor.PythonProcessorInitializationContext;
import org.apache.nifi.python.processor.PythonProcessorProxy;
+import org.apache.nifi.python.processor.RecordTransform;
import org.apache.nifi.python.processor.RecordTransformProxy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+
+import static org.apache.nifi.components.AsyncLoadedProcessor.LoadState;
public class StandardPythonProcessorBridge implements PythonProcessorBridge {
private static final Logger logger = LoggerFactory.getLogger(StandardPythonProcessorBridge.class);
private final PythonController controller;
+ private final ProcessorCreationWorkflow creationWorkflow;
+ private final PythonProcessorDetails processorDetails;
private volatile PythonProcessorAdapter adapter;
- private volatile Processor proxy;
- private final String processorType;
- private final String version;
+ private final PythonProcessorProxy proxy;
private final File workingDir;
private final File moduleFile;
private volatile long lastModified;
+ private volatile LoadState loadState = LoadState.DOWNLOADING_DEPENDENCIES;
private volatile PythonProcessorInitializationContext initializationContext;
+
private StandardPythonProcessorBridge(final Builder builder) {
this.controller = builder.controller;
- this.adapter = builder.adapter;
- this.processorType = builder.processorType;
- this.version = builder.version;
+ this.creationWorkflow = builder.creationWorkflow;
+ this.processorDetails = builder.processorDetails;
this.workingDir = builder.workDir;
this.moduleFile = builder.moduleFile;
this.lastModified = this.moduleFile.lastModified();
@@ -58,19 +65,57 @@ public class StandardPythonProcessorBridge implements PythonProcessorBridge {
}
@Override
- public PythonProcessorAdapter getProcessorAdapter() {
- return adapter;
+ public Optional<PythonProcessorAdapter> getProcessorAdapter() {
+ return Optional.ofNullable(adapter);
}
@Override
- public void initialize(final PythonProcessorInitializationContext context) {
+ public Future<Void> initialize(final PythonProcessorInitializationContext context) {
this.initializationContext = context;
- this.adapter.initialize(context);
+
+ final String threadName = "Initialize Python Processor %s (%s)".formatted(initializationContext.getIdentifier(), getProcessorType());
+ final CompletableFuture<Void> future = new CompletableFuture<>();
+
+ Thread.ofVirtual().name(threadName).start(() -> initializePythonSide(future));
+ return future;
+ }
+
+ public LoadState getLoadState() {
+ return loadState;
+ }
+
+ private void initializePythonSide(final CompletableFuture<Void> future) {
+ try {
+ try {
+ creationWorkflow.downloadDependencies();
+ loadState = LoadState.LOADING_PROCESSOR_CODE;
+ } catch (final Exception e) {
+ loadState = LoadState.DEPENDENCY_DOWNLOAD_FAILED;
+ throw e;
+ }
+
+ final PythonProcessorAdapter pythonProcessorAdapter;
+ try {
+ pythonProcessorAdapter = creationWorkflow.createProcessor();
+ pythonProcessorAdapter.initialize(initializationContext);
+ this.adapter = pythonProcessorAdapter;
+ this.proxy.onPythonSideInitialized(pythonProcessorAdapter);
+
+ loadState = LoadState.FINISHED_LOADING;
+ } catch (final Exception e) {
+ loadState = LoadState.LOADING_PROCESSOR_CODE_FAILED;
+ throw e;
+ }
+
+ future.complete(null);
+ } catch (final Throwable t) {
+ future.completeExceptionally(t);
+ }
}
@Override
public String getProcessorType() {
- return processorType;
+ return processorDetails.getProcessorType();
}
@Override
@@ -81,56 +126,49 @@ public class StandardPythonProcessorBridge implements PythonProcessorBridge {
@Override
public boolean reload() {
if (moduleFile.lastModified() <= lastModified) {
- logger.debug("Processor {} has not been modified since it was last loaded so will not reload", processorType);
+ logger.debug("Processor {} has not been modified since it was last loaded so will not reload", getProcessorType());
return false;
}
- controller.reloadProcessor(processorType, version, workingDir.getAbsolutePath());
- adapter = controller.createProcessor(processorType, version, workingDir.getAbsolutePath());
- adapter.initialize(initializationContext);
- proxy = createProxy();
+ controller.reloadProcessor(getProcessorType(), processorDetails.getProcessorVersion(), workingDir.getAbsolutePath());
+ initializePythonSide(new CompletableFuture<>());
lastModified = moduleFile.lastModified();
return true;
}
private PythonProcessorProxy createProxy() {
- final PythonProcessor pythonProcessor = adapter.getProcessor();
-
- // Instantiate the appropriate Processor Proxy based on the type of Python Processor
- if (pythonProcessor instanceof FlowFileTransform) {
+ final String implementedInterface = processorDetails.getInterface();
+ if (FlowFileTransform.class.getName().equals(implementedInterface)) {
return new FlowFileTransformProxy(this);
- } else {
+ }
+ if (RecordTransform.class.getName().equals(implementedInterface)) {
return new RecordTransformProxy(this);
}
+
+ throw new IllegalArgumentException("Python Processor does not implement any of the valid interfaces. Interface implemented: " + implementedInterface);
}
public static class Builder {
private PythonController controller;
- private PythonProcessorAdapter adapter;
- private String processorType;
- private String version;
+ private ProcessorCreationWorkflow creationWorkflow;
private File workDir;
private File moduleFile;
+ private PythonProcessorDetails processorDetails;
public Builder controller(final PythonController controller) {
this.controller = controller;
return this;
}
- public Builder processorAdapter(final PythonProcessorAdapter adapter) {
- this.adapter = adapter;
- return this;
- }
-
- public Builder processorType(final String type) {
- this.processorType = type;
+ public Builder creationWorkflow(final ProcessorCreationWorkflow creationWorkflow) {
+ this.creationWorkflow = creationWorkflow;
return this;
}
- public Builder processorVersion(final String version) {
- this.version = version;
+ public Builder processorDetails(final PythonProcessorDetails details) {
+ this.processorDetails = details;
return this;
}
@@ -148,11 +186,11 @@ public class StandardPythonProcessorBridge implements PythonProcessorBridge {
if (controller == null) {
throw new IllegalStateException("Must specify the PythonController");
}
- if (adapter == null) {
- throw new IllegalStateException("Must specify the Processor Adapter");
+ if (creationWorkflow == null) {
+ throw new IllegalStateException("Must specify the Processor Creation Workflow");
}
- if (processorType == null) {
- throw new IllegalStateException("Must specify the Processor Type");
+ if (processorDetails == null) {
+ throw new IllegalStateException("Must specify the Processor Details");
}
if (workDir == null) {
throw new IllegalStateException("Must specify the Working Directory");
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/FlowFileTransformProxy.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/FlowFileTransformProxy.java
index 8ebf00eb19..f482e6f27d 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/FlowFileTransformProxy.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/FlowFileTransformProxy.java
@@ -17,7 +17,6 @@
package org.apache.nifi.python.processor;
-import java.util.Map;
import org.apache.nifi.annotation.behavior.DefaultRunDuration;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
@@ -28,31 +27,30 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import py4j.Py4JNetworkException;
+import java.util.Map;
+import java.util.Optional;
+
@SupportsBatching(defaultDuration = DefaultRunDuration.TWENTY_FIVE_MILLIS)
public class FlowFileTransformProxy extends PythonProcessorProxy {
private final PythonProcessorBridge bridge;
private volatile FlowFileTransform transform;
-
public FlowFileTransformProxy(final PythonProcessorBridge bridge) {
super(bridge);
this.bridge = bridge;
- this.transform = (FlowFileTransform) bridge.getProcessorAdapter().getProcessor();
}
- protected void reloadProcessor() {
- final boolean reloaded = bridge.reload();
- if (reloaded) {
- transform = (FlowFileTransform) bridge.getProcessorAdapter().getProcessor();
- getLogger().info("Successfully reloaded Processor");
- }
- }
-
@OnScheduled
public void setContext(final ProcessContext context) {
- transform.setContext(context);
+ final Optional<PythonProcessorAdapter> optionalAdapter = bridge.getProcessorAdapter();
+ if (optionalAdapter.isEmpty()) {
+ throw new IllegalStateException(this + " is not finished initializing");
+ }
+
+ this.transform = (FlowFileTransform) optionalAdapter.get().getProcessor();
+ this.transform.setContext(context);
}
@Override
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java
index 987c75e074..e302ba75f9 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java
@@ -17,16 +17,9 @@
package org.apache.nifi.python.processor;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.AsyncLoadedProcessor;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
@@ -34,13 +27,21 @@ import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.Relationship;
-public abstract class PythonProcessorProxy extends AbstractProcessor {
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+public abstract class PythonProcessorProxy extends AbstractProcessor implements AsyncLoadedProcessor {
private final PythonProcessorBridge bridge;
private volatile Set<Relationship> cachedRelationships = null;
private volatile List<PropertyDescriptor> cachedPropertyDescriptors = null;
private volatile Map<String, PropertyDescriptor> cachedDynamicDescriptors = null;
- private boolean supportsDynamicProperties;
- private volatile boolean running = false;
+ private volatile boolean supportsDynamicProperties;
protected static final Relationship REL_ORIGINAL = new Relationship.Builder()
.name("original")
@@ -51,22 +52,37 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
.name("failure")
.description("The original FlowFile will be routed to this relationship if it unable to be transformed for some reason")
.build();
- private static final Set<Relationship> implicitRelationships = Collections.unmodifiableSet(
- new HashSet<>(Arrays.asList(REL_ORIGINAL, REL_FAILURE)));
+ private static final Set<Relationship> implicitRelationships = Set.of(
+ REL_ORIGINAL,
+ REL_FAILURE);
public PythonProcessorProxy(final PythonProcessorBridge bridge) {
this.bridge = bridge;
- supportsDynamicProperties = bridge.getProcessorAdapter().isDynamicPropertySupported();
+ }
+
+ public void onPythonSideInitialized(final PythonProcessorAdapter adapter) {
+ supportsDynamicProperties = adapter.isDynamicPropertySupported();
+ }
+
+ @Override
+ public LoadState getState() {
+ return bridge.getLoadState();
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- if (running && cachedPropertyDescriptors != null) {
+ if (cachedPropertyDescriptors != null) {
return this.cachedPropertyDescriptors;
}
+ final Optional<PythonProcessorAdapter> optionalAdapter = bridge.getProcessorAdapter();
+ if (optionalAdapter.isEmpty()) {
+ // If we don't have the adapter yet, use whatever is cached, even if it's old, or an empty List if we have nothing cached.
+ return this.cachedPropertyDescriptors == null ? Collections.emptyList() : cachedPropertyDescriptors;
+ }
+
try {
- final List<PropertyDescriptor> properties = bridge.getProcessorAdapter().getSupportedPropertyDescriptors();
+ final List<PropertyDescriptor> properties = optionalAdapter.get().getSupportedPropertyDescriptors();
this.cachedPropertyDescriptors = properties; // cache descriptors in case the processor is updated and the properties can no longer be properly accessed
return properties;
} catch (final Exception e) {
@@ -76,19 +92,27 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
}
}
-
@Override
protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ final Optional<PythonProcessorAdapter> optionalAdapter = bridge.getProcessorAdapter();
+ if (optionalAdapter.isEmpty()) {
+ return List.of(new ValidationResult.Builder()
+ .subject("Processor")
+ .explanation("Processor has not yet completed initialization")
+ .valid(false)
+ .build());
+ }
+
try {
reload();
- return bridge.getProcessorAdapter().customValidate(validationContext);
+ return optionalAdapter.get().customValidate(validationContext);
} catch (final Exception e) {
getLogger().warn("Failed to perform validation for Python Processor {}; assuming invalid", this, e);
return Collections.singleton(new ValidationResult.Builder()
.subject("Perform Validation")
.valid(false)
- .explanation("Unable to communicate with Python Processor")
+ .explanation("Failed to trigger Python Processor to perform validation: " + e)
.build());
}
}
@@ -106,7 +130,9 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
}
try {
- return bridge.getProcessorAdapter().getSupportedDynamicPropertyDescriptor(propertyDescriptorName);
+ final Optional<PythonProcessorAdapter> optionalAdapter = bridge.getProcessorAdapter();
+ return optionalAdapter.map(adapter -> adapter.getSupportedDynamicPropertyDescriptor(propertyDescriptorName))
+ .orElse(null);
} catch (final Exception e) {
getLogger().warn("Failed to obtain Dynamic Property Descriptor with name {} from Python Processor {}; assuming property is not valid", propertyDescriptorName, this, e);
return null;
@@ -138,12 +164,10 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
}
this.cachedDynamicDescriptors = dynamicDescriptors;
- this.running = true;
}
@OnStopped
public void destroyCachedElements() {
- this.running = false;
this.cachedRelationships = null;
this.cachedDynamicDescriptors = null;
}
@@ -161,11 +185,14 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
private Set<Relationship> fetchRelationshipsFromPythonProcessor() {
Set<Relationship> processorRelationships;
try {
- processorRelationships = bridge.getProcessorAdapter().getRelationships();
+ processorRelationships = bridge.getProcessorAdapter()
+ .map(PythonProcessorAdapter::getRelationships)
+ .orElseGet(HashSet::new);
} catch (final Exception e) {
getLogger().warn("Failed to obtain list of Relationships from Python Processor {}; assuming no explicit relationships", this, e);
processorRelationships = new HashSet<>();
}
+
processorRelationships.addAll(getImplicitRelationships());
return processorRelationships;
}
@@ -174,12 +201,16 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
@OnScheduled
public void onScheduled(final ProcessContext context) {
reload();
- bridge.getProcessorAdapter().onScheduled(context);
+ bridge.getProcessorAdapter()
+ .orElseThrow(() -> new IllegalStateException("Processor has not finished initializing"))
+ .onScheduled(context);
}
@OnStopped
public void onStopped(final ProcessContext context) {
- bridge.getProcessorAdapter().onStopped(context);
+ bridge.getProcessorAdapter()
+ .orElseThrow(() -> new IllegalStateException("Processor has not finished initializing"))
+ .onStopped(context);
}
@Override
@@ -188,11 +219,15 @@ public abstract class PythonProcessorProxy extends AbstractProcessor {
}
private void reload() {
- reloadProcessor();
- supportsDynamicProperties = bridge.getProcessorAdapter().isDynamicPropertySupported();
- }
+ final boolean reloaded = bridge.reload();
+ if (reloaded) {
+ getLogger().info("Successfully reloaded Processor");
+ }
- protected abstract void reloadProcessor();
+ supportsDynamicProperties = bridge.getProcessorAdapter()
+ .orElseThrow(() -> new IllegalStateException("Processor has not finished initializing"))
+ .isDynamicPropertySupported();
+ }
protected Set<Relationship> getImplicitRelationships() {
return implicitRelationships;
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/RecordTransformProxy.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/RecordTransformProxy.java
index a9e400a004..3339df9494 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/RecordTransformProxy.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/RecordTransformProxy.java
@@ -17,17 +17,6 @@
package org.apache.nifi.python.processor;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
import org.apache.nifi.NullSuppression;
import org.apache.nifi.annotation.behavior.DefaultRunDuration;
import org.apache.nifi.annotation.behavior.SupportsBatching;
@@ -55,12 +44,24 @@ import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordSchema;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
@SupportsBatching(defaultDuration = DefaultRunDuration.TWENTY_FIVE_MILLIS)
public class RecordTransformProxy extends PythonProcessorProxy {
private final PythonProcessorBridge bridge;
private volatile RecordTransform transform;
-
static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder()
.name("Record Reader")
.displayName("Record Reader")
@@ -80,7 +81,6 @@ public class RecordTransformProxy extends PythonProcessorProxy {
public RecordTransformProxy(final PythonProcessorBridge bridge) {
super(bridge);
this.bridge = bridge;
- this.transform = (RecordTransform) bridge.getProcessorAdapter().getProcessor();
}
@Override
@@ -92,18 +92,15 @@ public class RecordTransformProxy extends PythonProcessorProxy {
return properties;
}
-
- public void reloadProcessor() {
- final boolean reloaded = bridge.reload();
- if (reloaded) {
- transform = (RecordTransform) bridge.getProcessorAdapter().getProcessor();
- getLogger().info("Successfully reloaded Processor");
- }
- }
-
@OnScheduled
public void setProcessContext(final ProcessContext context) {
- transform.setContext(context);
+ final Optional<PythonProcessorAdapter> adapterOptional = bridge.getProcessorAdapter();
+ if (adapterOptional.isEmpty()) {
+ throw new IllegalStateException(this + " is not finished initializing");
+ }
+
+ this.transform = (RecordTransform) adapterOptional.get().getProcessor();
+ this.transform.setContext(context);
}
@Override
@@ -182,10 +179,10 @@ public class RecordTransformProxy extends PythonProcessorProxy {
session.transfer(flowFile, REL_FAILURE);
destinationTuples.values().forEach(tuple -> {
- session.remove(tuple.getFlowFile());
+ session.remove(tuple.flowFile());
try {
- tuple.getWriter().close();
+ tuple.writer().close();
} catch (final IOException ioe) {
getLogger().warn("Failed to close Record Writer for FlowFile created in this session", ioe);
}
@@ -217,26 +214,25 @@ public class RecordTransformProxy extends PythonProcessorProxy {
final Map<Relationship, List<FlowFile>> flowFilesPerRelationship = new HashMap<>();
for (final Map.Entry<RecordGroupingKey, DestinationTuple> entry : destinationTuples.entrySet()) {
final DestinationTuple destinationTuple = entry.getValue();
- final RecordSetWriter writer = destinationTuple.getWriter();
+ final RecordSetWriter writer = destinationTuple.writer();
final WriteResult writeResult = writer.finishRecordSet();
writer.close();
// Create attribute map
- final Map<String, String> attributes = new HashMap<>();
- attributes.putAll(writeResult.getAttributes());
+ final Map<String, String> attributes = new HashMap<>(writeResult.getAttributes());
attributes.put("record.count", String.valueOf(writeResult.getRecordCount()));
attributes.put("mime.type", writer.getMimeType());
final RecordGroupingKey groupingKey = entry.getKey();
- final Map<String, Object> partition = groupingKey.getPartition();
+ final Map<String, Object> partition = groupingKey.partition();
if (partition != null) {
partition.forEach((key, value) -> attributes.put(key, Objects.toString(value)));
}
// Update the FlowFile and add to the appropriate Relationship and grouping
- final FlowFile outputFlowFile = session.putAllAttributes(destinationTuple.getFlowFile(), attributes);
- final Relationship destinationRelationship = new Relationship.Builder().name(groupingKey.getRelationship()).build();
+ final FlowFile outputFlowFile = session.putAllAttributes(destinationTuple.flowFile(), attributes);
+ final Relationship destinationRelationship = new Relationship.Builder().name(groupingKey.relationship()).build();
final List<FlowFile> flowFiles = flowFilesPerRelationship.computeIfAbsent(destinationRelationship, key -> new ArrayList<>());
flowFiles.add(outputFlowFile);
}
@@ -289,7 +285,7 @@ public class RecordTransformProxy extends PythonProcessorProxy {
}
// Transform the result into a Record and write it out
- destinationTuple.getWriter().write(transformed);
+ destinationTuple.writer().write(transformed);
}
@@ -326,59 +322,13 @@ public class RecordTransformProxy extends PythonProcessorProxy {
* A tuple representing the name of a Relationship to which a Record should be transferred and an optional Partition that may distinguish
* a Record from other Records going to the same Relationship
*/
- private static class RecordGroupingKey {
- private final String relationship;
- private final Map<String, Object> partition;
-
- public RecordGroupingKey(final String relationship, final Map<String, Object> partition) {
- this.relationship = relationship;
- this.partition = partition;
- }
-
- public String getRelationship() {
- return relationship;
- }
-
- public Map<String, Object> getPartition() {
- return partition;
- }
+ private record RecordGroupingKey(String relationship, Map<String, Object> partition) {
- @Override
- public boolean equals(final Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- final RecordGroupingKey that = (RecordGroupingKey) o;
- return Objects.equals(relationship, that.relationship) && Objects.equals(partition, that.partition);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(relationship, partition);
- }
}
/**
* A tuple of a FlowFile and the RecordSetWriter to use for writing to that FlowFile
*/
- private static class DestinationTuple {
- private final FlowFile flowFile;
- private final RecordSetWriter writer;
-
- public DestinationTuple(final FlowFile flowFile, final RecordSetWriter writer) {
- this.flowFile = flowFile;
- this.writer = writer;
- }
-
- public FlowFile getFlowFile() {
- return flowFile;
- }
-
- public RecordSetWriter getWriter() {
- return writer;
- }
+ private record DestinationTuple(FlowFile flowFile, RecordSetWriter writer) {
}
}
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java
index 5d586d51d3..e8369f450d 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java
@@ -74,6 +74,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -476,7 +478,7 @@ public class PythonControllerInteractionIT {
runnerV1.getFlowFilesForRelationship("success").get(0).assertContentEquals("Hello, World");
// Create an instance of WriteMessage V2
- final PythonProcessorBridge procV2 = bridge.createProcessor(createId(), "WriteMessage", "0.0.2-SNAPSHOT", true);
+ final PythonProcessorBridge procV2 = createProcessor("WriteMessage", "0.0.2-SNAPSHOT");
final FlowFileTransformProxy wrapperV2 = new FlowFileTransformProxy(procV2);
final TestRunner runnerV2 = TestRunners.newTestRunner(wrapperV2);
runnerV2.enqueue("");
@@ -515,7 +517,7 @@ public class PythonControllerInteractionIT {
// Create a Record to transform and transform it
final String json = "[{ \"name\": \"John Doe\" }]";
final RecordSchema schema = createSimpleRecordSchema("name");
- final RecordTransform recordTransform = (RecordTransform) processor.getProcessorAdapter().getProcessor();
+ final RecordTransform recordTransform = (RecordTransform) processor.getProcessorAdapter().get().getProcessor();
recordTransform.setContext(context);
final RecordTransformResult result = recordTransform.transformRecord(json, schema, new EmptyAttributeMap()).get(0);
@@ -562,7 +564,7 @@ public class PythonControllerInteractionIT {
// Create a Record to transform and transform it
final String json = "[{\"name\": \"Jake Doe\", \"father\": { \"name\": \"John Doe\" }}]";
final RecordSchema recordSchema = createTwoLevelRecord().getSchema();
- final RecordTransform recordTransform = (RecordTransform) processor.getProcessorAdapter().getProcessor();
+ final RecordTransform recordTransform = (RecordTransform) processor.getProcessorAdapter().get().getProcessor();
recordTransform.setContext(context);
final RecordTransformResult result = recordTransform.transformRecord(json, recordSchema, new EmptyAttributeMap()).get(0);
@@ -574,10 +576,12 @@ public class PythonControllerInteractionIT {
@Test
- public void testLogger() {
+ public void testLogger() throws ExecutionException, InterruptedException {
bridge.discoverExtensions();
final String procId = createId();
+ // Create the Processor, but we do not use this.createProcessor() because we want to explicitly have access to the
+ // initialization context to inject in the logger that we want.
final PythonProcessorBridge logContentsBridge = bridge.createProcessor(procId, "LogContents", VERSION, true);
final ComponentLog logger = Mockito.mock(ComponentLog.class);
@@ -593,7 +597,7 @@ public class PythonControllerInteractionIT {
}
};
- logContentsBridge.initialize(initContext);
+ logContentsBridge.initialize(initContext).get();
final TestRunner runner = TestRunners.newTestRunner(logContentsBridge.getProcessorProxy());
runner.enqueue("Hello World");
@@ -668,9 +672,22 @@ public class PythonControllerInteractionIT {
}
private PythonProcessorBridge createProcessor(final String type) {
+ return createProcessor(type, VERSION);
+ }
+
+ private PythonProcessorBridge createProcessor(final String type, final String version) {
bridge.discoverExtensions();
- final PythonProcessorBridge processor = bridge.createProcessor(createId(), type, VERSION, true);
- processor.initialize(createInitContext());
+ final PythonProcessorBridge processor = bridge.createProcessor(createId(), type, version, true);
+ final Future<Void> future = processor.initialize(createInitContext());
+
+ try {
+ future.get();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e.getCause());
+ }
+
return processor;
}
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/properties.py b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/properties.py
index 885447914c..e0c773bd27 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/properties.py
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/properties.py
@@ -259,34 +259,34 @@ class PropertyDescriptor:
return values
- def __add_resource_definition(self, gateway, resouce_definition, builder):
+ def __add_resource_definition(self, gateway, resource_definition, builder):
allowed_types = 0
- if resouce_definition.allow_file:
+ if resource_definition.allow_file:
allowed_types += 1
- if resouce_definition.allow_directory:
+ if resource_definition.allow_directory:
allowed_types += 1
- if resouce_definition.allow_url:
+ if resource_definition.allow_url:
allowed_types += 1
- if resouce_definition.allow_text:
+ if resource_definition.allow_text:
allowed_types += 1
array_type = gateway.jvm.org.apache.nifi.components.resource.ResourceType
types = gateway.new_array(array_type, allowed_types)
index = 0
- if resouce_definition.allow_file:
+ if resource_definition.allow_file:
types[index] = gateway.jvm.org.apache.nifi.components.resource.ResourceType.FILE
index += 1
- if resouce_definition.allow_directory:
+ if resource_definition.allow_directory:
types[index] = gateway.jvm.org.apache.nifi.components.resource.ResourceType.DIRECTORY
index += 1
- if resouce_definition.allow_url:
+ if resource_definition.allow_url:
types[index] = gateway.jvm.org.apache.nifi.components.resource.ResourceType.URL
index += 1
- if resouce_definition.allow_text:
+ if resource_definition.allow_text:
types[index] = gateway.jvm.org.apache.nifi.components.resource.ResourceType.TEXT
index += 1
- cardinality = gateway.jvm.org.apache.nifi.components.resource.ResourceCardinality.MULTIPLE if resouce_definition.allow_multiple else \
+ cardinality = gateway.jvm.org.apache.nifi.components.resource.ResourceCardinality.MULTIPLE if resource_definition.allow_multiple else \
gateway.jvm.org.apache.nifi.components.resource.ResourceCardinality.SINGLE
builder.identifiesExternalResource(cardinality, types[0], types[1:])
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/pom.xml b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/pom.xml
index 4c5f54c953..77023a817d 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/pom.xml
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/pom.xml
@@ -30,6 +30,10 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-api</artifactId>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java
index 16bbe6c83f..080d890ede 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java
@@ -45,6 +45,16 @@ public interface PythonController {
*/
void discoverExtensions(List<String> directories, String workDirectory);
+
+ /**
+ * Downloads any third-party dependencies that are necessary in order to use the processor of the given type and version
+ *
+ * @param type the type of the processor
+ * @param version the version of the processor
+ * @param workDirectory the work directory where the processor's virtual environment should live
+ */
+ void downloadDependencies(String type, String version, String workDirectory);
+
/**
* Creates a Processor of the given type and version, returning a PythonProcessorAdapter that can be used for interacting with it.
*
@@ -79,4 +89,12 @@ public interface PythonController {
*/
String getModuleFile(String processorType, String version);
+ /**
+ * Returns details about the Processor with the given type and version
+ *
+ * @param type the type of the Processor
+ * @param version the version of the Processor
+ * @return the details that have been discovered
+ */
+ PythonProcessorDetails getProcessorDetails(String type, String version);
}
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessorDetails.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessorDetails.java
index 758e338929..8a311c8089 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessorDetails.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessorDetails.java
@@ -54,4 +54,9 @@ public interface PythonProcessorDetails {
* @return the dependencies that must be imported in order to use the Processor
*/
List<String> getDependencies();
+
+ /**
+ * @return the name of the Java interface that is implemented by the Python Processor
+ */
+ String getInterface();
}
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorBridge.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorBridge.java
index 219164ee70..7d08bdfae3 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorBridge.java
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorBridge.java
@@ -17,20 +17,26 @@
package org.apache.nifi.python.processor;
+import org.apache.nifi.components.AsyncLoadedProcessor.LoadState;
import org.apache.nifi.processor.Processor;
+import java.util.Optional;
+import java.util.concurrent.Future;
+
/**
* A model object that is used to bridge the gap between what is necessary for the Framework to interact
* with a Python Processor and what is made available by lower level APIs
*/
public interface PythonProcessorBridge {
/**
- * @return a proxy for the PythonProcessorAdapter that is responsible for calling the associated method on the Python side
+ * @return a proxy for the PythonProcessorAdapter that is responsible for calling the associated method on the Python side,
+ * or an empty Optional if the adapter has not yet been initialized
*/
- PythonProcessorAdapter getProcessorAdapter();
+ Optional<PythonProcessorAdapter> getProcessorAdapter();
/**
- * @return a proxy for the actual Processor implementation that will trigger the appropriate method on the Python side
+ * @return a proxy for the actual Processor implementation that will trigger the appropriate method on the Python side, or an empty Optional
+ * if the processor/adapter have not yet been initialized
*/
Processor getProcessorProxy();
@@ -51,6 +57,10 @@ public interface PythonProcessorBridge {
* Initializes the Processor
* @param context the initialization context
*/
- void initialize(PythonProcessorInitializationContext context);
+ Future<Void> initialize(PythonProcessorInitializationContext context);
+ /**
+ * @return the current state of the Processor loading
+ */
+ LoadState getLoadState();
}
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py
index ab6a3e6144..be4d53271d 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py
@@ -48,6 +48,20 @@ class Controller:
def discoverExtensions(self, dirs, work_dir):
self.extensionManager.discoverExtensions(dirs, work_dir)
+ def getProcessorDetails(self, type, version):
+ processor_details = self.extensionManager.get_processor_details(type, version)
+ if processor_details is None:
+ raise ValueError(f"Invalid Processor Type/Version: {type}/{version}")
+
+ return processor_details
+
+ def downloadDependencies(self, type, version, work_dir):
+ processor_details = self.extensionManager.get_processor_details(type, version)
+ if processor_details is None:
+ raise ValueError(f"Invalid Processor Type/Version: {type}/{version}")
+
+ self.extensionManager.import_external_dependencies(processor_details, work_dir)
+
def createProcessor(self, processorType, version, work_dir):
processorClass = self.extensionManager.getProcessorClass(processorType, version, work_dir)
processor = processorClass(jvm=self.gateway.jvm)
@@ -61,14 +75,6 @@ class Controller:
module_file = self.extensionManager.get_module_file(processorType, version)
return module_file
- def getProcessorDependencies(self, processorType, version):
- deps = self.extensionManager.__get_dependencies_for_extension_type(processorType, version)
- dependencyList = self.gateway.jvm.java.util.ArrayList()
- for dep in deps:
- dependencyList.add(dep)
-
- return dependencyList
-
def setGateway(self, gateway):
self.gateway = gateway
self.extensionManager = ExtensionManager.ExtensionManager(gateway)
diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
index 72d2ede34e..37bba85fa2 100644
--- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
+++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
@@ -26,6 +26,7 @@ from pathlib import Path
logger = logging.getLogger("org.apache.nifi.py4j.ExtensionManager")
+
# A simple wrapper class to encompass a processor type and its version
class ExtensionId:
def __init__(self, classname=None, version=None):
@@ -43,14 +44,19 @@ class ExtensionDetails:
class Java:
implements = ['org.apache.nifi.python.PythonProcessorDetails']
- def __init__(self, gateway, type, version='Unknown', dependencies=None, source_location=None, package_name=None, description=None, tags=None):
+ def __init__(self, gateway, type, interfaces, version='Unknown', dependencies=None, source_location=None, package_name=None, description=None, tags=None):
self.gateway = gateway
+ self.type = type
+
+ if interfaces is None:
+ interfaces = []
+ self.interfaces = interfaces
+
if dependencies is None:
dependencies = []
if tags is None:
tags = []
- self.type = type
self.version = version
self.dependencies = dependencies
self.source_location = source_location
@@ -87,6 +93,10 @@ class ExtensionDetails:
return list
+ def getInterface(self):
+ if len(self.interfaces) == 0:
+ return None
+ return self.interfaces[0]
@@ -111,6 +121,11 @@ class ExtensionManager:
self.gateway = gateway
+ def get_processor_details(self, classname, version):
+ extension_id = ExtensionId(classname=classname, version=version)
+ return self.processor_details.get(extension_id)
+
+
def getProcessorTypes(self):
"""
:return: a list of Processor types that have been discovered by the #discoverExtensions method
@@ -230,7 +245,6 @@ class ExtensionManager:
logger.error("Failed to load Python extensions from module file {0}. This module will be ignored.".format(module_file), exc_info=True)
-
def __gather_extension_details(self, module_file, work_dir, local_dependencies=None):
path = Path(module_file)
basename = os.path.basename(module_file)
@@ -276,9 +290,6 @@ class ExtensionManager:
self.module_files_by_extension_type[id] = module_file
- # If there are any dependencies, use pip to install them
- self.__import_external_dependencies(classname, details.dependencies, work_dir, details.version)
-
def __get_download_complete_marker_file(self, work_dir, extension_type, version):
return os.path.join(work_dir, 'extensions', extension_type, version, 'dependency-download.complete')
@@ -299,9 +310,9 @@ class ExtensionManager:
class_nodes = self.__get_class_nodes(root_node)
for class_node in class_nodes:
- logger.debug("Checking if class %s is a processor" % class_node.name)
+ logger.debug(f"Checking if class {class_node.name} is a processor")
if self.__is_processor_class_node(class_node):
- logger.info("Discovered Processor class {0} in module {1}".format(class_node.name, module_file))
+ logger.info(f"Discovered Processor class {class_node.name} in module {module_file}")
details = self.__get_processor_details(class_node, module_file)
details_by_class[class_node.name] = details
@@ -323,34 +334,50 @@ class ExtensionManager:
"""
# Look for a 'Java' sub-class
+ interfaces = self.__get_java_interfaces(class_node)
+ return len(interfaces) > 0
+
+
+ def __get_java_interfaces(self, class_node):
+ # Get all class definition nodes
child_class_nodes = self.__get_class_nodes(class_node)
+ interfaces = []
for child_class_node in child_class_nodes:
- if child_class_node.name == 'Java':
- # Look for an assignment that assigns values to the `implements` keyword
- assignment_nodes = self.__get_assignment_nodes(child_class_node)
- for assignment_node in assignment_nodes:
- if (len(assignment_node.targets) == 1 and assignment_node.targets[0].id == 'implements'):
- assigned_values = assignment_node.value.elts
- for assigned_value in assigned_values:
- if assigned_value.value in self.processorInterfaces:
- return True
- return False
+ # Look for a 'Java' sub-class
+ if child_class_node.name != 'Java':
+ continue
+
+ # Look for an assignment that assigns values to the `implements` keyword
+ assignment_nodes = self.__get_assignment_nodes(child_class_node)
+ for assignment_node in assignment_nodes:
+ targets = assignment_node.targets
+ if (len(targets) == 1 and targets[0].id == 'implements'):
+ assigned_values = assignment_node.value.elts
+ for assigned_value in assigned_values:
+ if assigned_value.value in self.processorInterfaces:
+ interfaces.append(assigned_value.value)
+
+ return interfaces
def __get_processor_details(self, class_node, module_file):
# Look for a 'ProcessorDetails' class
child_class_nodes = self.__get_class_nodes(class_node)
+ # Get the Java interfaces that it implements
+ interfaces = self.__get_java_interfaces(class_node)
+
for child_class_node in child_class_nodes:
if child_class_node.name == 'ProcessorDetails':
- logger.debug('Found ProcessorDetails class in %s' % class_node.name)
+ logger.debug(f"Found ProcessorDetails class in {class_node.name}")
version = self.__get_processor_version(child_class_node, class_node.name)
dependencies = self.__get_processor_dependencies(child_class_node, class_node.name)
description = self.__get_processor_description(child_class_node, class_node.name)
tags = self.__get_processor_tags(child_class_node, class_node.name)
return ExtensionDetails(gateway=self.gateway,
+ interfaces=interfaces,
type=class_node.name,
version=version,
dependencies=dependencies,
@@ -359,6 +386,7 @@ class ExtensionManager:
tags=tags)
return ExtensionDetails(gateway=self.gateway,
+ interfaces=interfaces,
type=class_node.name,
version='Unknown',
dependencies=[],
@@ -368,7 +396,8 @@ class ExtensionManager:
def __get_processor_version(self, details_node, class_name):
assignment_nodes = self.__get_assignment_nodes(details_node)
for assignment_node in assignment_nodes:
- if (len(assignment_node.targets) == 1 and assignment_node.targets[0].id == 'version'):
+ targets = assignment_node.targets
+ if (len(targets) == 1 and targets[0].id == 'version'):
assigned_values = assignment_node.value.value
logger.info("Found version of {0} to be {1}".format(class_name, assigned_values))
return assigned_values
@@ -395,7 +424,8 @@ class ExtensionManager:
def __get_assigned_list(self, details_node, class_name, element_name):
assignment_nodes = self.__get_assignment_nodes(details_node)
for assignment_node in assignment_nodes:
- if (len(assignment_node.targets) == 1 and assignment_node.targets[0].id == element_name):
+ targets = assignment_node.targets
+ if (len(targets) == 1 and targets[0].id == element_name):
assigned_values = assignment_node.value.elts
declared_dependencies = []
for assigned_value in assigned_values:
@@ -410,7 +440,8 @@ class ExtensionManager:
def __get_processor_description(self, details_node, class_name):
assignment_nodes = self.__get_assignment_nodes(details_node)
for assignment_node in assignment_nodes:
- if (len(assignment_node.targets) == 1 and assignment_node.targets[0].id == 'description'):
+ targets = assignment_node.targets
+ if (len(targets) == 1 and targets[0].id == 'description'):
return assignment_node.value.value
# No description found
@@ -429,7 +460,10 @@ class ExtensionManager:
return assignment_nodes
- def __import_external_dependencies(self, class_name, dependencies, work_dir, extension_version):
+ def import_external_dependencies(self, processor_details, work_dir):
+ class_name = processor_details.getProcessorType()
+ extension_version = processor_details.getProcessorVersion()
+
completion_marker_file = self.__get_download_complete_marker_file(work_dir, class_name, extension_version)
target_dir = os.path.dirname(completion_marker_file)
@@ -440,32 +474,26 @@ class ExtensionManager:
logger.info("All dependencies have already been imported for {0}".format(class_name))
return True
- success = False
+ dependencies = processor_details.getDependencies()
if len(dependencies) > 0:
python_cmd = os.getenv("PYTHON_CMD")
args = [python_cmd, '-m', 'pip', 'install', '--target', target_dir]
for dep in dependencies:
args.append(dep)
- logger.info("Importing dependencies {0} for {1} to {2} using command {3}".format(dependencies, class_name, target_dir, args))
+ logger.info(f"Importing dependencies {dependencies} for {class_name} to {target_dir} using command {args}")
result = subprocess.run(args)
if result.returncode == 0:
- logger.info("Successfully imported requirements for {0} to {1}".format(class_name, target_dir))
- success = True
+ logger.info(f"Successfully imported requirements for {class_name} to {target_dir}")
else:
- logger.error("Failed to import requirements for {0}: process exited with status code {1}".format(class_name, result))
- return False
+ raise RuntimeError(f"Failed to import requirements for {class_name}: process exited with status code {result}")
else:
- logger.info("No dependencies to import for {0}".format(class_name))
- success = True
-
- if success:
- # Write a completion Marker File
- with open(completion_marker_file, "w") as file:
- file.write("True")
+ logger.info(f"No dependencies to import for {class_name}")
- return success
+ # Write a completion Marker File
+ with open(completion_marker_file, "w") as file:
+ file.write("True")
def __load_extension_module(self, file, local_dependencies):
@@ -475,7 +503,7 @@ class ExtensionManager:
if local_dependency == file:
continue
- logger.debug("Loading local dependency {0} before loading {1}".format(local_dependency, file))
+ logger.debug(f"Loading local dependency {local_dependency} before loading {file}")
self.__load_extension_module(local_dependency, None)
@@ -502,14 +530,14 @@ class ExtensionManager:
# Load the module
sys.modules[moduleName] = module
moduleSpec.loader.exec_module(module)
- logger.info("Loaded module %s" % moduleName)
+ logger.info(f"Loaded module {moduleName}")
# Find the Processor class and return it
for name, member in inspect.getmembers(module):
if inspect.isclass(member):
- logger.debug('Found class: %s' % member)
+ logger.debug(f"Found class: {member}")
if self.__is_processor_class(member):
- logger.debug('Found Processor: %s' % member)
+ logger.debug(f"Found Processor: {member}")
return member
return None
@@ -536,6 +564,6 @@ class ExtensionManager:
# The class implements something. Check if it implements Processor
for interface in instance.implements:
if interface in self.processorInterfaces:
- logger.debug('%s implements Processor' % potentialProcessorClass)
+ logger.debug(f"{potentialProcessorClass} implements Processor")
return True
return False
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/python/PythonProcessorIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/python/PythonProcessorIT.java
index 1b98c4bc51..f3452f9974 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/python/PythonProcessorIT.java
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/python/PythonProcessorIT.java
@@ -139,7 +139,7 @@ public class PythonProcessorIT extends NiFiSystemIT {
final String headerLine = lines[0];
final List<String> headers = Stream.of(headerLine.split(","))
.map(String::trim)
- .collect(Collectors.toList());
+ .toList();
assertTrue(headers.contains("name"));
assertTrue(headers.contains("age"));
assertTrue(headers.contains("color"));
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties
index d454c4b3b5..0069027b3e 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties
@@ -263,3 +263,5 @@ nifi.kerberos.spnego.authentication.expiration=12 hours
# external properties files for variable registry
# supports a comma delimited list of file locations
nifi.variable.registry.properties=
+
+nifi.flow.analysis.background.task.schedule=5 mins