You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by fg...@apache.org on 2025/10/06 14:07:22 UTC
(nifi-minifi-cpp) 01/03: MINIFICPP-2645 Enable NiFi python processors using both EL and validators
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 4d7c19eaac51f535016ba00c473dcb178cfdeec1
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Oct 2 17:03:34 2025 +0200
MINIFICPP-2645 Enable NiFi python processors using both EL and validators
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
Closes #2041
---
docker/test/integration/cluster/ImageStore.py | 4 +-
docker/test/integration/features/python.feature | 20 ++++++++
.../ExpressionLanguagePropertyWithValidator.py | 26 +++++++++++
.../ExpressionLanguagePropertyWithValidator.py | 53 ++++++++++++++++++++++
.../pythonprocessors/nifiapi/processorbase.py | 9 +++-
5 files changed, 109 insertions(+), 3 deletions(-)
diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py
index 1b0802df2..9fe772dcb 100644
--- a/docker/test/integration/cluster/ImageStore.py
+++ b/docker/test/integration/cluster/ImageStore.py
@@ -213,6 +213,7 @@ class ImageStore:
COPY TestStateManager.py {minifi_python_dir}/nifi_python_processors/TestStateManager.py
COPY NifiStyleLogDynamicProperties.py {minifi_python_dir}/nifi_python_processors/NifiStyleLogDynamicProperties.py
COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py
+ COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py
RUN python3 -m venv {minifi_python_venv_parent}/venv
""".format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION,
pip3_install_command=pip3_install_command,
@@ -237,7 +238,8 @@ class ImageStore:
build_full_python_resource_path("SetRecordField.py"),
build_full_python_resource_path("TestStateManager.py"),
build_full_python_resource_path("NifiStyleLogDynamicProperties.py"),
- build_full_python_resource_path("LogDynamicProperties.py")
+ build_full_python_resource_path("LogDynamicProperties.py"),
+ build_full_python_resource_path("ExpressionLanguagePropertyWithValidator.py")
])
def __build_minifi_cpp_image_with_llamacpp_model(self):
diff --git a/docker/test/integration/features/python.feature b/docker/test/integration/features/python.feature
index 966f9d81b..08b8dd1c4 100644
--- a/docker/test/integration/features/python.feature
+++ b/docker/test/integration/features/python.feature
@@ -269,3 +269,23 @@ Feature: MiNiFi can use python processors in its flows
When all instances start up
Then the Minifi logs contain the following message: "Static Property value: static value" in less than 60 seconds
And the Minifi logs contain the following message: "Dynamic Property value: dynamic value" in less than 60 seconds
+
+ Scenario: MiNiFi C++ allows NiFi python processors to use property validators for expression language enabled properties
+ Given a GenerateFlowFile processor with the "File Size" property set to "0B"
+ And a ExpressionLanguagePropertyWithValidator processor with the "Integer Property" property set to "42"
+ And the "success" relationship of the GenerateFlowFile processor is connected to the ExpressionLanguagePropertyWithValidator
+ And python processors without dependencies are present on the MiNiFi agent
+
+ When all instances start up
+ Then the Minifi logs contain the following message: "Integer Property value: 42" in less than 60 seconds
+
+ Scenario: MiNiFi C++ rolls back session if expression language cannot be evaluated as integer in NiFi python processors
+ Given a GenerateFlowFile processor with the "File Size" property set to "0B"
+ And a UpdateAttribute processor with the "my.integer" property set to "invalid"
+ And a ExpressionLanguagePropertyWithValidator processor with the "Integer Property" property set to "${my.integer}"
+ And the "success" relationship of the GenerateFlowFile processor is connected to the UpdateAttribute
+ And the "success" relationship of the UpdateAttribute processor is connected to the ExpressionLanguagePropertyWithValidator
+ And python processors without dependencies are present on the MiNiFi agent
+
+ When all instances start up
+ Then the Minifi logs contain the following message: "ProcessSession rollback for ExpressionLanguagePropertyWithValidator" in less than 60 seconds
diff --git a/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py b/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py
new file mode 100644
index 000000000..42bdede34
--- /dev/null
+++ b/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from ..core.Processor import Processor
+
+
+class ExpressionLanguagePropertyWithValidator(Processor):
+ def __init__(self, context):
+ super(ExpressionLanguagePropertyWithValidator, self).__init__(
+ context=context,
+ clazz='ExpressionLanguagePropertyWithValidator',
+ class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.',
+ properties={},
+ schedule={'scheduling strategy': 'EVENT_DRIVEN'},
+ auto_terminate=[])
diff --git a/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py b/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py
new file mode 100644
index 000000000..346f1ede4
--- /dev/null
+++ b/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult
+from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators
+
+
+class ExpressionLanguagePropertyWithValidator(FlowFileTransform):
+
+ class Java:
+ implements = ['org.apache.nifi.python.processor.FlowFileTransform']
+
+ class ProcessorDetails:
+ version = '1.2.3'
+ description = "Test processor"
+ dependencies = []
+
+ INTEGER_PROPERTY = PropertyDescriptor(
+ name="Integer Property",
+ description="A dummy integer property",
+ required=True,
+ validators=[StandardValidators.INTEGER_VALIDATOR],
+ expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES
+ )
+
+ def __init__(self, **kwargs):
+ pass
+
+ def getPropertyDescriptors(self):
+ return [self.INTEGER_PROPERTY]
+
+ def getDynamicPropertyDescriptor(self, propertyname):
+ return PropertyDescriptor(name=propertyname,
+ description="A user-defined property",
+ dynamic=True)
+
+ def transform(self, context, flow_file):
+ integer_value = context.getProperty(self.INTEGER_PROPERTY).asInteger()
+ self.logger.info("Integer Property value: {}".format(integer_value))
+
+ return FlowFileTransformResult('success', contents="content")
diff --git a/extensions/python/pythonprocessors/nifiapi/processorbase.py b/extensions/python/pythonprocessors/nifiapi/processorbase.py
index 7a884aeed..1ef376c4a 100644
--- a/extensions/python/pythonprocessors/nifiapi/processorbase.py
+++ b/extensions/python/pythonprocessors/nifiapi/processorbase.py
@@ -15,7 +15,7 @@
from abc import ABC, abstractmethod
from typing import List
-from .properties import ExpressionLanguageScope, PropertyDescriptor, translateStandardValidatorToMiNiFiPropertype
+from .properties import ExpressionLanguageScope, PropertyDescriptor, translateStandardValidatorToMiNiFiPropertype, MinifiPropertyTypes
from .properties import ProcessContext as ProcessContextProxy
from minifi_native import OutputStream, Processor, ProcessContext, ProcessSession
@@ -54,8 +54,13 @@ class ProcessorBase(ABC):
self.supports_dynamic_properties = False
for property in self.getPropertyDescriptors():
- property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators)
expression_language_supported = True if property.expressionLanguageScope != ExpressionLanguageScope.NONE else False
+ property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators)
+
+ # MiNiFi C++ does not support validators for expression language enabled properties
+ if expression_language_supported and property_type_code is not None and property_type_code != MinifiPropertyTypes.NON_BLANK_TYPE:
+ self.logger.warn("Property '{}' has validators defined, but since it also supports Expression Language, the validators will be ignored.".format(property.name))
+ property_type_code = None
# MiNiFi C++ does not support dependant properties, so if a property depends on another property, it should not be required
is_required = True if property.required and not property.dependencies else False