You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by bb...@apache.org on 2024/11/06 19:19:56 UTC

(nifi) branch main updated: NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)

This is an automated email from the ASF dual-hosted git repository.

bbende pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git


The following commit(s) were added to refs/heads/main by this push:
     new ffe2649955 NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)
ffe2649955 is described below

commit ffe26499558a7325d1704094aac520178904a955
Author: Zoltán Kornél Török <ta...@gmail.com>
AuthorDate: Wed Nov 6 20:19:51 2024 +0100

    NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)
---
 .../src/main/python/src/nifiapi/documentation.py   |  28 +++++-
 ...rtyDescription.java => PropertyDependency.java} |  18 +---
 .../documentation/PropertyDescription.java         |   2 +
 .../main/python/framework/ProcessorInspection.py   | 100 ++++++++++++++-------
 .../manifest/StandardRuntimeManifestService.java   |  21 +++++
 5 files changed, 122 insertions(+), 47 deletions(-)

diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py
index 6bce8beb64..bc52711888 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py
@@ -95,6 +95,27 @@ class MultiProcessorUseCaseDetails:
     def __str__(self):
         return f"MultiProcessorUseCaseDetails[description={self.description}]"
 
+class PropertyDependency:
+    class Java:
+            implements = ['org.apache.nifi.python.processor.documentation.PropertyDependency']
+
+    def __init__(self,
+                      name: str,
+                      display_name: str,
+                      dependent_values: list[str]):
+             self.name = name
+             self.display_name = display_name
+             self.dependent_values = dependent_values
+
+    def getName(self):
+        return self.name
+
+    def getDisplayName(self):
+        return self.display_name
+
+    def getDependentValues(self):
+        return ArrayList(self.dependent_values)
+
 class PropertyDescription:
     class Java:
         implements = ['org.apache.nifi.python.processor.documentation.PropertyDescription']
@@ -108,7 +129,8 @@ class PropertyDescription:
                  default_value: str = None,
                  expression_language_scope: str = None,
                  controller_service_definition: str = None,
-                 allowable_values: list[str] = None):
+                 allowable_values: list[str] = None,
+                 dependencies: list[PropertyDependency] = None):
         self.name = name
         self.description = description
         self.display_name = display_name
@@ -118,6 +140,7 @@ class PropertyDescription:
         self.expression_language_scope = expression_language_scope
         self.controller_service_definition = controller_service_definition
         self.allowable_values = allowable_values if allowable_values is not None else []
+        self.dependencies = dependencies if dependencies is not None else []
 
     def getName(self):
         return self.name
@@ -145,3 +168,6 @@ class PropertyDescription:
 
     def getAllowableValues(self):
         return ArrayList(self.allowable_values)
+
+    def getDependencies(self):
+        return ArrayList(self.dependencies)
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java
similarity index 76%
copy from nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
copy to nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java
index 0b32ac78c3..36eb4ea766 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java
@@ -18,24 +18,12 @@
 package org.apache.nifi.python.processor.documentation;
 
 import java.util.List;
+
 import org.apache.nifi.python.PythonObjectProxy;
 
-public interface PropertyDescription extends PythonObjectProxy {
+public interface PropertyDependency extends PythonObjectProxy {
     String getName();
 
     String getDisplayName();
-
-    String getDescription();
-
-    String getDefaultValue();
-
-    boolean isSensitive();
-
-    boolean isRequired();
-
-    String getExpressionLanguageScope();
-
-    String getControllerServiceDefinition();
-
-    List<String> getAllowableValues();
+    List<String> getDependentValues();
 }
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
index 0b32ac78c3..8ea5765a19 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java
@@ -38,4 +38,6 @@ public interface PropertyDescription extends PythonObjectProxy {
     String getControllerServiceDefinition();
 
     List<String> getAllowableValues();
+
+    List<PropertyDependency> getDependencies();
 }
diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py
index 10df6ca056..d9907a39d7 100644
--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py
+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py
@@ -18,7 +18,7 @@ import logging
 import textwrap
 import os
 import BundleCoordinate
-from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription
+from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription, PropertyDependency
 
 import ExtensionDetails
 
@@ -47,6 +47,71 @@ class StringConstantVisitor(ast.NodeVisitor):
                             self.string_assignments.append[variable_name] = string_value
         self.generic_visit(node)
 
+class CollectPropertyDescriptorVisitors(ast.NodeVisitor):
+
+    def __init__(self, module_string_constants, processor_name):
+        self.module_string_constants = module_string_constants
+        self.discovered_property_descriptors = {}
+        self.processor_name = processor_name
+        self.logger = logging.getLogger("python.CollectPropertyDescriptorVisitors")
+
+    def resolve_dependencies(self, node: ast.AST):
+        resolved_dependencies = []
+        for dependency in node.elts:
+            variable_name = dependency.args[0].id
+            if not self.discovered_property_descriptors[variable_name]:
+                self.logger.error(f"Not able to find actual property descriptor for {variable_name}, so not able to resolve property dependencies in {self.processor_name}.")
+            else:
+                actual_property = self.discovered_property_descriptors[variable_name]
+                dependent_values = []
+                for dependent_value in dependency.args[1:]:
+                    dependent_values.append(get_constant_values(dependent_value, self.module_string_constants))
+                resolved_dependencies.append(PropertyDependency(name = actual_property.name,
+                                                                display_name = actual_property.display_name,
+                                                                dependent_values = dependent_values))
+        return resolved_dependencies
+
+    def resolve_property_descriptor_name_in_code(self, node: ast.AST):
+        if isinstance(node.targets[0], ast.Name):
+            return node.targets[0].id
+        elif isinstance(node.targets[0], ast.Attribute):
+            return node.targets[0].attr
+        else:
+            raise Exception("Unable to determine name from source code")
+
+    def visit_Assign(self, node: ast.AST):
+        if self.assignment_is_property_descriton(node):
+            property_descriptor_name_in_code = self.resolve_property_descriptor_name_in_code(node)
+            self.logger.debug(f"Found PropertyDescriptor in the following assignment {property_descriptor_name_in_code}")
+            if not node.value.keywords:
+                self.logger.error(f"Not able to parse {property_descriptor_name_in_code} PropertyDescriptor as no keywords assignments used.")
+            else:
+                descriptor_info = {}
+                for keyword in node.value.keywords:
+                    key = keyword.arg
+                    if key == 'dependencies':
+                        self.logger.debug(f"Resolving dependencies for {property_descriptor_name_in_code}.")
+                        value = self.resolve_dependencies(keyword.value)
+                    else:
+                        value = get_constant_values(keyword.value, self.module_string_constants)
+                    descriptor_info[key] = value
+
+                self.discovered_property_descriptors[property_descriptor_name_in_code] = PropertyDescription(name=descriptor_info.get('name'),
+                                                                                        description=descriptor_info.get('description'),
+                                                                                         display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
+                                                                                         required=replace_null(descriptor_info.get('required'), False),
+                                                                                         sensitive=replace_null(descriptor_info.get('sensitive'), False),
+                                                                                         default_value=descriptor_info.get('default_value'),
+                                                                                         expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
+                                                                                            controller_service_definition=descriptor_info.get('controller_service_definition'),
+                                                                                          allowable_values = descriptor_info.get('allowable_values'),
+                                                                                          dependencies = descriptor_info.get('dependencies'))
+        self.generic_visit(node)
+
+
+    def assignment_is_property_descriton(self, node: ast.AST):
+        return isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == 'PropertyDescriptor'
+
 
 def get_module_string_constants(module_file: str) -> dict:
     with open(module_file) as file:
@@ -219,36 +284,9 @@ def get_processor_configurations(constructor_calls: ast.List) -> list:
 
 
 def get_property_descriptions(class_node, module_string_constants):
-    descriptions = []
-
-    for element in class_node.body:
-        if not isinstance(element, ast.Assign) or not element.value:
-            continue
-        if not isinstance(element.value, ast.Call):
-            continue
-        if element.value.func.id != 'PropertyDescriptor':
-            continue
-        if not element.value.keywords:
-            continue
-
-        descriptor_info = {}
-        for keyword in element.value.keywords:
-            key = keyword.arg
-            value = get_constant_values(keyword.value, module_string_constants)
-            descriptor_info[key] = value
-
-        description = PropertyDescription(name=descriptor_info.get('name'),
-                                        description=descriptor_info.get('description'),
-                                        display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
-                                        required=replace_null(descriptor_info.get('required'), False),
-                                        sensitive=replace_null(descriptor_info.get('sensitive'), False),
-                                        default_value=descriptor_info.get('default_value'),
-                                        expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
-                                        controller_service_definition=descriptor_info.get('controller_service_definition'),
-                                        allowable_values = descriptor_info.get('allowable_values'))
-        descriptions.append(description)
-
-    return descriptions
+    visitor = CollectPropertyDescriptorVisitors(module_string_constants, class_node.name)
+    visitor.visit(class_node)
+    return visitor.discovered_property_descriptors.values()
 
 
 def replace_null(val: any, replacement: any):
diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
index 524c232dc7..be2a2c4875 100644
--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java
@@ -23,6 +23,8 @@ import org.apache.nifi.c2.protocol.component.api.BuildInfo;
 import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
 import org.apache.nifi.extension.manifest.AllowableValue;
 import org.apache.nifi.extension.manifest.ControllerServiceDefinition;
+import org.apache.nifi.extension.manifest.Dependency;
+import org.apache.nifi.extension.manifest.DependentValues;
 import org.apache.nifi.extension.manifest.ExpressionLanguageScope;
 import org.apache.nifi.extension.manifest.Extension;
 import org.apache.nifi.extension.manifest.ExtensionManifest;
@@ -240,9 +242,28 @@ public class StandardRuntimeManifestService implements RuntimeManifestService {
         property.setControllerServiceDefinition(getManifestControllerServiceDefinition(propertyDescription.getControllerServiceDefinition()));
         property.setAllowableValues(getAllowableValues(propertyDescription));
 
+        property.setDependencies(getDependencies(propertyDescription));
+
         return property;
     }
 
+    private static List<Dependency> getDependencies(org.apache.nifi.python.processor.documentation.PropertyDescription propertyDescription) {
+        return Optional.ofNullable(propertyDescription.getDependencies()).orElse(List.of())
+                .stream()
+                .map(value -> {
+                    DependentValues dependentValues = new DependentValues();
+                    dependentValues.setValues(value.getDependentValues());
+
+                    Dependency dependency = new Dependency();
+                    dependency.setPropertyName(value.getName());
+                    dependency.setPropertyDisplayName(value.getDisplayName());
+                    dependency.setDependentValues(dependentValues);
+
+                    return dependency;
+                })
+                .toList();
+    }
+
     private static ControllerServiceDefinition getManifestControllerServiceDefinition(final String controllerServiceClassName) {
         if (controllerServiceClassName == null) {
             return null;