You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by bb...@apache.org on 2024/11/06 19:19:56 UTC
(nifi) branch main updated: NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)
This is an automated email from the ASF dual-hosted git repository.
bbende pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new ffe2649955 NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)
ffe2649955 is described below
commit ffe26499558a7325d1704094aac520178904a955
Author: Zoltán Kornél Török <ta...@gmail.com>
AuthorDate: Wed Nov 6 20:19:51 2024 +0100
NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)
---
.../src/main/python/src/nifiapi/documentation.py | 28 +++++-
...rtyDescription.java => PropertyDependency.java} | 18 +---
.../documentation/PropertyDescription.java | 2 +
.../main/python/framework/ProcessorInspection.py | 100 ++++++++++++++-------
.../manifest/StandardRuntimeManifestService.java | 21 +++++
5 files changed, 122 insertions(+), 47 deletions(-)
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py
index 6bce8beb64..bc52711888 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py
@@ -95,6 +95,27 @@ class MultiProcessorUseCaseDetails:
def __str__(self):
return f"MultiProcessorUseCaseDetails[description={self.description}]"
+class PropertyDependency:
+ class Java:
+ implements = ['org.apache.nifi.python.processor.documentation.PropertyDependency']
+
+ def __init__(self,
+ name: str,
+ display_name: str,
+ dependent_values: list[str]):
+ self.name = name
+ self.display_name = display_name
+ self.dependent_values = dependent_values
+
+ def getName(self):
+ return self.name
+
+ def getDisplayName(self):
+ return self.display_name
+
+ def getDependentValues(self):
+ return ArrayList(self.dependent_values)
+
class PropertyDescription:
class Java:
implements = ['org.apache.nifi.python.processor.documentation.PropertyDescription']
@@ -108,7 +129,8 @@ class PropertyDescription:
default_value: str = None,
expression_language_scope: str = None,
controller_service_definition: str = None,
- allowable_values: list[str] = None):
+ allowable_values: list[str] = None,
+ dependencies: list[PropertyDependency] = None):
self.name = name
self.description = description
self.display_name = display_name
@@ -118,6 +140,7 @@ class PropertyDescription:
self.expression_language_scope = expression_language_scope
self.controller_service_definition = controller_service_definition
self.allowable_values = allowable_values if allowable_values is not None else []
+ self.dependencies = dependencies if dependencies is not None else []
def getName(self):
return self.name
@@ -145,3 +168,6 @@ class PropertyDescription:
def getAllowableValues(self):
return ArrayList(self.allowable_values)
+
+ def getDependencies(self):
+ return ArrayList(self.dependencies)
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java
similarity index 76%
copy from nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
copy to nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java
index 0b32ac78c3..36eb4ea766 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java
@@ -18,24 +18,12 @@
package org.apache.nifi.python.processor.documentation;
import java.util.List;
+
import org.apache.nifi.python.PythonObjectProxy;
-public interface PropertyDescription extends PythonObjectProxy {
+public interface PropertyDependency extends PythonObjectProxy {
String getName();
String getDisplayName();
-
- String getDescription();
-
- String getDefaultValue();
-
- boolean isSensitive();
-
- boolean isRequired();
-
- String getExpressionLanguageScope();
-
- String getControllerServiceDefinition();
-
- List<String> getAllowableValues();
+ List<String> getDependentValues();
}
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
index 0b32ac78c3..8ea5765a19 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
@@ -38,4 +38,6 @@ public interface PropertyDescription extends PythonObjectProxy {
String getControllerServiceDefinition();
List<String> getAllowableValues();
+
+ List<PropertyDependency> getDependencies();
}
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py
index 10df6ca056..d9907a39d7 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py
@@ -18,7 +18,7 @@ import logging
import textwrap
import os
import BundleCoordinate
-from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription
+from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription, PropertyDependency
import ExtensionDetails
@@ -47,6 +47,71 @@ class StringConstantVisitor(ast.NodeVisitor):
self.string_assignments.append[variable_name] = string_value
self.generic_visit(node)
+class CollectPropertyDescriptorVisitors(ast.NodeVisitor):
+
+ def __init__(self, module_string_constants, processor_name):
+ self.module_string_constants = module_string_constants
+ self.discovered_property_descriptors = {}
+ self.processor_name = processor_name
+ self.logger = logging.getLogger("python.CollectPropertyDescriptorVisitors")
+
+ def resolve_dependencies(self, node: ast.AST):
+ resolved_dependencies = []
+ for dependency in node.elts:
+ variable_name = dependency.args[0].id
+ if not self.discovered_property_descriptors[variable_name]:
+ self.logger.error(f"Not able to find actual property descriptor for {variable_name}, so not able to resolve property dependencies in {self.processor_name}.")
+ else:
+ actual_property = self.discovered_property_descriptors[variable_name]
+ dependent_values = []
+ for dependent_value in dependency.args[1:]:
+ dependent_values.append(get_constant_values(dependent_value, self.module_string_constants))
+ resolved_dependencies.append(PropertyDependency(name = actual_property.name,
+ display_name = actual_property.display_name,
+ dependent_values = dependent_values))
+ return resolved_dependencies
+
+ def resolve_property_descriptor_name_in_code(self, node: ast.AST):
+ if isinstance(node.targets[0], ast.Name):
+ return node.targets[0].id
+ elif isinstance(node.targets[0], ast.Attribute):
+ return node.targets[0].attr
+ else:
+ raise Exception("Unable to determine name from source code")
+
+ def visit_Assign(self, node: ast.AST):
+ if self.assignment_is_property_descriton(node):
+ property_descriptor_name_in_code = self.resolve_property_descriptor_name_in_code(node)
+ self.logger.debug(f"Found PropertyDescriptor in the following assignment {property_descriptor_name_in_code}")
+ if not node.value.keywords:
+ self.logger.error(f"Not able to parse {property_descriptor_name_in_code} PropertyDescriptor as no keywords assignments used.")
+ else:
+ descriptor_info = {}
+ for keyword in node.value.keywords:
+ key = keyword.arg
+ if key == 'dependencies':
+ self.logger.debug(f"Resolving dependencies for {property_descriptor_name_in_code}.")
+ value = self.resolve_dependencies(keyword.value)
+ else:
+ value = get_constant_values(keyword.value, self.module_string_constants)
+ descriptor_info[key] = value
+
+ self.discovered_property_descriptors[property_descriptor_name_in_code] = PropertyDescription(name=descriptor_info.get('name'),
+ description=descriptor_info.get('description'),
+ display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
+ required=replace_null(descriptor_info.get('required'), False),
+ sensitive=replace_null(descriptor_info.get('sensitive'), False),
+ default_value=descriptor_info.get('default_value'),
+ expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
+ controller_service_definition=descriptor_info.get('controller_service_definition'),
+ allowable_values = descriptor_info.get('allowable_values'),
+ dependencies = descriptor_info.get('dependencies'))
+ self.generic_visit(node)
+
+
+ def assignment_is_property_descriton(self, node: ast.AST):
+ return isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == 'PropertyDescriptor'
+
def get_module_string_constants(module_file: str) -> dict:
with open(module_file) as file:
@@ -219,36 +284,9 @@ def get_processor_configurations(constructor_calls: ast.List) -> list:
def get_property_descriptions(class_node, module_string_constants):
- descriptions = []
-
- for element in class_node.body:
- if not isinstance(element, ast.Assign) or not element.value:
- continue
- if not isinstance(element.value, ast.Call):
- continue
- if element.value.func.id != 'PropertyDescriptor':
- continue
- if not element.value.keywords:
- continue
-
- descriptor_info = {}
- for keyword in element.value.keywords:
- key = keyword.arg
- value = get_constant_values(keyword.value, module_string_constants)
- descriptor_info[key] = value
-
- description = PropertyDescription(name=descriptor_info.get('name'),
- description=descriptor_info.get('description'),
- display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
- required=replace_null(descriptor_info.get('required'), False),
- sensitive=replace_null(descriptor_info.get('sensitive'), False),
- default_value=descriptor_info.get('default_value'),
- expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
- controller_service_definition=descriptor_info.get('controller_service_definition'),
- allowable_values = descriptor_info.get('allowable_values'))
- descriptions.append(description)
-
- return descriptions
+ visitor = CollectPropertyDescriptorVisitors(module_string_constants, class_node.name)
+ visitor.visit(class_node)
+ return visitor.discovered_property_descriptors.values()
def replace_null(val: any, replacement: any):
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
index 524c232dc7..be2a2c4875 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
@@ -23,6 +23,8 @@ import org.apache.nifi.c2.protocol.component.api.BuildInfo;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import org.apache.nifi.extension.manifest.AllowableValue;
import org.apache.nifi.extension.manifest.ControllerServiceDefinition;
+import org.apache.nifi.extension.manifest.Dependency;
+import org.apache.nifi.extension.manifest.DependentValues;
import org.apache.nifi.extension.manifest.ExpressionLanguageScope;
import org.apache.nifi.extension.manifest.Extension;
import org.apache.nifi.extension.manifest.ExtensionManifest;
@@ -240,9 +242,28 @@ public class StandardRuntimeManifestService implements RuntimeManifestService {
property.setControllerServiceDefinition(getManifestControllerServiceDefinition(propertyDescription.getControllerServiceDefinition()));
property.setAllowableValues(getAllowableValues(propertyDescription));
+ property.setDependencies(getDependencies(propertyDescription));
+
return property;
}
+ private static List<Dependency> getDependencies(org.apache.nifi.python.processor.documentation.PropertyDescription propertyDescription) {
+ return Optional.ofNullable(propertyDescription.getDependencies()).orElse(List.of())
+ .stream()
+ .map(value -> {
+ DependentValues dependentValues = new DependentValues();
+ dependentValues.setValues(value.getDependentValues());
+
+ Dependency dependency = new Dependency();
+ dependency.setPropertyName(value.getName());
+ dependency.setPropertyDisplayName(value.getDisplayName());
+ dependency.setDependentValues(dependentValues);
+
+ return dependency;
+ })
+ .toList();
+ }
+
private static ControllerServiceDefinition getManifestControllerServiceDefinition(final String controllerServiceClassName) {
if (controllerServiceClassName == null) {
return null;