You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2025/12/01 16:30:07 UTC
(nifi-minifi-cpp) branch main updated: MINIFICPP-2650 - Processor c api
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new ef3dd5656 MINIFICPP-2650 - Processor c api
ef3dd5656 is described below
commit ef3dd5656957b39a39edc0bb7f3f426ccc33a1e1
Author: Adam Debreceni <ad...@apache.org>
AuthorDate: Mon Dec 1 12:49:37 2025 +0100
MINIFICPP-2650 - Processor c api
Closes #1987
Signed-off-by: Martin Zink <ma...@apache.org>
---
Extensions.md | 6 +-
cmake/Extensions.cmake | 58 +++
core-framework/CMakeLists.txt | 4 +-
core-framework/common/CMakeLists.txt | 22 ++
.../{ => common}/include/core/ClassName.h | 0
.../include/core/PropertyDefinitionBuilder.h | 0
.../{ => common}/include/io/InputStream.h | 0
.../{ => common}/include/io/OutputStream.h | 0
core-framework/{ => common}/include/io/Stream.h | 0
.../{ => common}/include/utils/ArrayUtils.h | 0
.../{ => common}/include/utils/ClassUtils.h | 0
core-framework/{ => common}/include/utils/Enum.h | 9 +-
.../{ => common}/include/utils/Environment.h | 0
.../{ => common}/include/utils/FailurePolicy.h | 0
.../{ => common}/include/utils/GeneralUtils.h | 0
core-framework/{ => common}/include/utils/Hash.h | 0
core-framework/common/include/utils/IdHash.h | 40 ++
.../{ => common}/include/utils/OptionalUtils.h | 0
.../{ => common}/include/utils/ParsingErrors.h | 0
.../{ => common}/include/utils/ParsingUtils.h | 0
.../{ => common}/include/utils/PropertyErrors.h | 0
.../{ => common}/include/utils/SmallString.h | 0
.../{ => common}/include/utils/StringUtils.h | 1 +
.../{ => common}/include/utils/TimeUtil.h | 7 -
.../common/include/utils/UnicodeConversion.h | 37 ++
.../utils/detail/MonadicOperationWrappers.h | 0
.../{ => common}/include/utils/expected.h | 0
.../{ => common}/include/utils/meta/detected.h | 0
core-framework/{ => common}/include/utils/span.h | 0
core-framework/{ => common}/src/io/InputStream.cpp | 0
.../{ => common}/src/io/OutputStream.cpp | 0
.../{ => common}/src/utils/Environment.cpp | 0
core-framework/common/src/utils/Id.cpp | 133 +++++++
.../{ => common}/src/utils/ParsingUtils.cpp | 0
.../{ => common}/src/utils/StringUtils.cpp | 16 +
core-framework/{ => common}/src/utils/TimeUtil.cpp | 9 -
core-framework/include/agent/agent_docs.h | 21 +-
.../include/core/ConfigurableComponentImpl.h | 1 +
core-framework/include/core/ObjectFactory.h | 2 +-
core-framework/include/core/ProcessorFactoryImpl.h | 2 +-
core-framework/include/core/ProcessorImpl.h | 7 +-
core-framework/include/io/BaseStream.h | 4 +-
core-framework/include/io/CRCStream.h | 4 +-
core-framework/include/io/StreamSlice.h | 2 +-
core-framework/include/io/ZlibStream.h | 4 +-
core-framework/include/utils/Id.h | 22 +-
core-framework/include/utils/Locations.h | 2 +-
.../include/utils/TimeZoneUtils.h | 17 +-
core-framework/include/utils/file/FileUtils.h | 9 +
.../include/utils/net/NetworkInterfaceInfo.h | 2 +-
.../src/core/ConfigurableComponentImpl.cpp | 9 +
core-framework/src/core/ProcessorImpl.cpp | 11 +-
core-framework/src/utils/Id.cpp | 106 ------
.../utils/TimeZoneUtils.cpp} | 30 +-
core-framework/src/utils/file/FileUtils.cpp | 25 ++
extension-framework/CMakeLists.txt | 2 +
.../cpp-extension-lib}/CMakeLists.txt | 22 +-
.../cpp-extension-lib/include/api/core/FlowFile.h | 22 +-
.../include/api/core/ProcessContext.h | 46 +++
.../include/api/core/ProcessSession.h | 54 +++
.../include/api/core/ProcessorImpl.h | 100 +++++
.../include/api/core/PublishedMetrics.h | 17 +-
.../cpp-extension-lib/include/api/core/Resource.h | 151 ++++++++
.../include/api/core/logging/Logger.h | 26 +-
.../cpp-extension-lib/include/api/utils/Export.h | 18 +-
.../include/api/utils/ProcessorConfigUtils.h | 170 +++++++++
.../include/api/utils/minifi-c-utils.h | 95 +++++
.../cpp-extension-lib/libtest}/CMakeLists.txt | 23 +-
.../libtest/CProcessorTestUtils.h | 38 ++
.../cpp-extension-lib/src/core/ProcessContext.cpp | 51 +++
.../cpp-extension-lib/src/core/ProcessSession.cpp | 153 ++++++++
.../cpp-extension-lib}/src/core/ProcessorImpl.cpp | 53 +--
.../cpp-extension-lib/src/core/logging/Logger.cpp | 69 ++++
.../cpp-extension-lib/src/utils/ParsingErrors.cpp | 19 +-
.../cpp-extension-lib/src/utils/PropertyErrors.cpp | 19 +-
.../cpp-extension-lib/src/utils/minifi-c-utils.cpp | 61 +++
extension-framework/include/core/ProcessSession.h | 1 -
.../include/utils/ExtensionInitUtils.h | 4 +-
.../src/utils/file/FileWriterCallback.cpp | 3 +
extensions/llamacpp/CMakeLists.txt | 4 +-
.../llamacpp/processors/DefaultLlamaContext.cpp | 6 +-
.../llamacpp/processors/ExtensionInitializer.cpp | 43 +++
.../llamacpp/processors/RunLlamaCppInference.cpp | 85 ++---
.../llamacpp/processors/RunLlamaCppInference.h | 51 +--
extensions/llamacpp/tests/CMakeLists.txt | 1 +
.../llamacpp/tests/RunLlamaCppInferenceTests.cpp | 39 +-
extensions/lua/LuaScriptFlowFile.cpp | 4 +-
extensions/lua/LuaScriptFlowFile.h | 2 +-
extensions/opencv/OpenCVLoader.cpp | 8 +-
extensions/python/ExecutePythonProcessor.cpp | 11 +-
extensions/python/ExecutePythonProcessor.h | 2 +-
extensions/python/PythonCreator.h | 15 +-
extensions/python/PythonDependencyInstaller.cpp | 2 +-
extensions/python/PythonDependencyInstaller.h | 2 +-
.../python/pythonlibloader/PythonLibLoader.cpp | 10 +-
extensions/python/pythonloader/PyProcLoader.cpp | 10 +-
extensions/python/types/PyScriptFlowFile.cpp | 3 +-
extensions/sftp/SFTPLoader.cpp | 8 +-
extensions/sql/processors/QueryDatabaseTable.h | 2 +-
.../processors/EvaluateJsonPath.h | 3 +-
.../standard-processors/processors/RetryFlowFile.h | 2 +-
.../standard-processors/processors/RouteText.h | 2 +-
.../standard-processors/processors/SplitJson.h | 1 +
.../tests/integration/TailFileIntegrationTest.cpp | 1 +
.../controllers/NetworkPrioritizerService.h | 1 +
libminifi/include/core/FlowFile.h | 4 +-
libminifi/include/core/ProcessSession.h | 2 +-
libminifi/include/core/Processor.h | 4 +-
libminifi/include/core/extension/Utils.h | 30 +-
libminifi/include/core/logging/LoggerBase.h | 8 +-
libminifi/include/core/logging/alert/AlertSink.h | 1 +
libminifi/include/utils/CProcessor.h | 176 +++++++++
libminifi/include/utils/FlowFileQueue.h | 3 +-
libminifi/src/core/ProcessSession.cpp | 1 +
libminifi/src/core/Processor.cpp | 9 +-
libminifi/src/core/extension/ExtensionManager.cpp | 5 +-
libminifi/src/core/extension/Utils.cpp | 65 ++++
libminifi/src/core/logging/LoggerBase.cpp | 11 +
libminifi/src/expression-language/Expression.cpp | 14 +-
libminifi/src/minifi-c.cpp | 410 ++++++++++++++++++++-
libminifi/src/utils/CProcessor.cpp | 45 +++
libminifi/src/utils/TimeUtil.cpp | 1 +
.../integration/ConnectionCountingServer.cpp | 1 +
libminifi/test/unit/CronTests.cpp | 8 +-
libminifi/test/unit/ExtensionVerificationTests.cpp | 19 +-
libminifi/test/unit/FileUtilsTests.cpp | 11 +
libminifi/test/unit/SchedulingAgentTests.cpp | 4 +-
libminifi/test/unit/StringUtilsTests.cpp | 10 +
minifi-api/CMakeLists.txt | 23 +-
.../{ => common}/include/minifi-cpp/Exception.h | 0
.../include/minifi-cpp/core/Annotation.h | 0
.../minifi-cpp/core/DynamicPropertyDefinition.h} | 4 +-
.../minifi-cpp/core/OutputAttributeDefinition.h | 0
.../include/minifi-cpp/core/ProcessorMetadata.h | 0
.../include/minifi-cpp/core/PropertyDefinition.h | 2 +-
.../include/minifi-cpp/core/PropertyValidator.h | 0
.../include/minifi-cpp/core/Relationship.h | 4 +
.../minifi-cpp/core/RelationshipDefinition.h | 0
.../include/minifi-cpp/core/SpecialFlowAttribute.h | 37 ++
.../include/minifi-cpp/core/logging/Logger.h | 33 +-
.../include/minifi-cpp/io/InputStream.h | 0
.../include/minifi-cpp/io/OutputStream.h | 2 +-
.../{ => common}/include/minifi-cpp/io/Stream.h | 0
.../include/minifi-cpp/io/StreamCallback.h | 0
.../{ => common}/include/minifi-cpp/utils/Id.h | 0
.../include/minifi-cpp/utils/Literals.h | 0
.../include/minifi-cpp/utils/SmallString.h | 0
.../{ => common}/include/minifi-cpp/utils/gsl.h | 0
minifi-api/include/minifi-c/minifi-c.h | 186 +++++++++-
minifi-api/include/minifi-cpp/Connection.h | 2 +-
minifi-api/include/minifi-cpp/agent/agent_docs.h | 11 +-
.../minifi-cpp/core/ConfigurableComponent.h | 2 +-
minifi-api/include/minifi-cpp/core/Connectable.h | 2 +-
...{ProcessorMetadata.h => ControllerServiceApi.h} | 19 +-
.../include/minifi-cpp/core/DynamicProperty.h | 15 +-
minifi-api/include/minifi-cpp/core/FlowFile.h | 22 +-
.../{ProcessorMetadata.h => OutputAttribute.h} | 22 +-
.../include/minifi-cpp/core/ProcessContext.h | 6 +-
.../include/minifi-cpp/core/ProcessSession.h | 5 +-
minifi-api/include/minifi-cpp/core/ProcessorApi.h | 9 +-
.../include/minifi-cpp/core/ProcessorDescriptor.h | 1 +
minifi-api/include/minifi-cpp/core/Property.h | 7 +-
.../minifi-cpp/core/logging/LoggerFactory.h | 2 +-
minifi-api/include/minifi-cpp/io/ArchiveStream.h | 4 +-
minifi-api/include/minifi-cpp/io/BaseStream.h | 4 +-
minifi-api/include/minifi-cpp/utils/TimeUtil.h | 23 +-
minifi-api/minifi-c-api.def | 25 ++
minifi_main/AgentDocs.cpp | 4 +-
168 files changed, 2843 insertions(+), 632 deletions(-)
diff --git a/Extensions.md b/Extensions.md
index 55ff97840..7d3fd5392 100644
--- a/Extensions.md
+++ b/Extensions.md
@@ -45,9 +45,11 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
.name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
.version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
.deinit = nullptr,
- .user_data = nullptr
+ .user_data = nullptr,
+ .processors_count = 0,
+ .processors_ptr = nullptr
};
- return MinifiCreateExtension(&ext_create_info);
+ return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
```
diff --git a/cmake/Extensions.cmake b/cmake/Extensions.cmake
index 82a66e14b..5786967a8 100644
--- a/cmake/Extensions.cmake
+++ b/cmake/Extensions.cmake
@@ -100,6 +100,64 @@ macro(register_extension extension-name extension-display-name extension-guard d
endif()
endmacro()
+macro(register_c_api_extension extension-name extension-display-name extension-guard description)
+ set(${extension-guard} ${extension-name} PARENT_SCOPE)
+ get_property(extensions GLOBAL PROPERTY EXTENSION-OPTIONS)
+ set_property(GLOBAL APPEND PROPERTY EXTENSION-OPTIONS ${extension-name})
+ target_compile_definitions(${extension-name}
+ PRIVATE "EXTENSION_NAME=${extension-name}" "EXTENSION_VERSION=${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}")
+ set_target_properties(${extension-name} PROPERTIES
+ ENABLE_EXPORTS True
+ POSITION_INDEPENDENT_CODE ON)
+ if(WIN32)
+ set_target_properties(${extension-name} PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
+ WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
+ else()
+ set_target_properties(${extension-name} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+ endif()
+
+ if (${ARGC} GREATER 5)
+ set(ARG_FLAGS ${ARGV5})
+ endif()
+
+ if (NOT "CI_ONLY_INSTALL" IN_LIST ARG_FLAGS OR CI_BUILD)
+ get_component_name(${extension-name} component-name)
+
+ if(WIN32)
+ install(TARGETS ${extension-name} RUNTIME DESTINATION extensions COMPONENT ${component-name})
+ else()
+ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND NOT APPLE)
+ target_link_options(${extension-name} PRIVATE "-Wl,--disable-new-dtags")
+ endif()
+ if (APPLE)
+ set_target_properties(${extension-name} PROPERTIES INSTALL_RPATH "@loader_path")
+ else()
+ set_target_properties(${extension-name} PROPERTIES INSTALL_RPATH "$ORIGIN")
+ endif()
+ if (MINIFI_PACKAGING_TYPE STREQUAL "RPM")
+ install(TARGETS ${extension-name}
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/extensions/
+ COMPONENT ${component-name})
+ set(RPM_EXPECTED_EXTENSION_LIST ${RPM_EXPECTED_EXTENSION_LIST} /usr/${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/extensions/lib${extension-name}.so)
+ set(RPM_EXPECTED_EXTENSION_LIST ${RPM_EXPECTED_EXTENSION_LIST} PARENT_SCOPE)
+ elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ")
+ install(TARGETS ${extension-name} LIBRARY DESTINATION extensions COMPONENT ${component-name})
+ else()
+ message(FATAL_ERROR "Invalid MINIFI_PACKAGING_TYPE")
+ endif()
+ endif()
+ endif()
+
+ ADD_FEATURE_INFO("${extension-display-name}" ${extension-guard} "${description}")
+ mark_as_advanced(${extension-guard})
+ # check for test directory
+ if(${ARGC} GREATER 4 AND NOT "${ARGV4}" STREQUAL "")
+ register_extension_test(${ARGV4})
+ endif()
+endmacro()
+
### TESTING MACROS
define_property(GLOBAL PROPERTY EXTENSION-TESTS
diff --git a/core-framework/CMakeLists.txt b/core-framework/CMakeLists.txt
index cf3203f24..757d087a8 100644
--- a/core-framework/CMakeLists.txt
+++ b/core-framework/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(common)
+
file(GLOB SOURCES
src/*.cpp
src/core/*.cpp
@@ -13,7 +15,7 @@ file(GLOB SOURCES
add_minifi_library(minifi-core-framework STATIC ${SOURCES})
target_include_directories(minifi-core-framework PUBLIC include)
-target_link_libraries(minifi-core-framework PUBLIC minifi-api ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON)
+target_link_libraries(minifi-core-framework PUBLIC minifi-api minifi-core-framework-common ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON)
if(NOT WIN32)
target_link_libraries(minifi-core-framework PUBLIC OSSP::libuuid++)
endif()
diff --git a/core-framework/common/CMakeLists.txt b/core-framework/common/CMakeLists.txt
new file mode 100644
index 000000000..928256c6b
--- /dev/null
+++ b/core-framework/common/CMakeLists.txt
@@ -0,0 +1,22 @@
+file(GLOB SOURCES
+ src/*.cpp
+ src/core/*.cpp
+ src/core/extension/*.cpp
+ src/io/*.cpp
+ src/http/*.cpp
+ src/utils/*.cpp
+ src/utils/crypto/*.cpp
+ src/utils/crypto/ciphers/*.cpp
+ src/utils/crypto/property_encryption/*.cpp
+ src/utils/net/*.cpp
+ src/utils/file/*.cpp)
+
+add_minifi_library(minifi-core-framework-common STATIC ${SOURCES})
+target_include_directories(minifi-core-framework-common PUBLIC include)
+target_link_libraries(minifi-core-framework-common PUBLIC minifi-api-common ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON)
+if(NOT WIN32)
+ target_link_libraries(minifi-core-framework-common PUBLIC OSSP::libuuid++)
+endif()
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
+ target_link_libraries(minifi-core-framework-common PUBLIC stdc++fs)
+endif()
diff --git a/core-framework/include/core/ClassName.h b/core-framework/common/include/core/ClassName.h
similarity index 100%
rename from core-framework/include/core/ClassName.h
rename to core-framework/common/include/core/ClassName.h
diff --git a/core-framework/include/core/PropertyDefinitionBuilder.h b/core-framework/common/include/core/PropertyDefinitionBuilder.h
similarity index 100%
rename from core-framework/include/core/PropertyDefinitionBuilder.h
rename to core-framework/common/include/core/PropertyDefinitionBuilder.h
diff --git a/core-framework/include/io/InputStream.h b/core-framework/common/include/io/InputStream.h
similarity index 100%
rename from core-framework/include/io/InputStream.h
rename to core-framework/common/include/io/InputStream.h
diff --git a/core-framework/include/io/OutputStream.h b/core-framework/common/include/io/OutputStream.h
similarity index 100%
copy from core-framework/include/io/OutputStream.h
copy to core-framework/common/include/io/OutputStream.h
diff --git a/core-framework/include/io/Stream.h b/core-framework/common/include/io/Stream.h
similarity index 100%
rename from core-framework/include/io/Stream.h
rename to core-framework/common/include/io/Stream.h
diff --git a/core-framework/include/utils/ArrayUtils.h b/core-framework/common/include/utils/ArrayUtils.h
similarity index 100%
rename from core-framework/include/utils/ArrayUtils.h
rename to core-framework/common/include/utils/ArrayUtils.h
diff --git a/core-framework/include/utils/ClassUtils.h b/core-framework/common/include/utils/ClassUtils.h
similarity index 100%
rename from core-framework/include/utils/ClassUtils.h
rename to core-framework/common/include/utils/ClassUtils.h
diff --git a/core-framework/include/utils/Enum.h b/core-framework/common/include/utils/Enum.h
similarity index 88%
rename from core-framework/include/utils/Enum.h
rename to core-framework/common/include/utils/Enum.h
index 070c926c1..c4695fa1b 100644
--- a/core-framework/include/utils/Enum.h
+++ b/core-framework/common/include/utils/Enum.h
@@ -18,9 +18,9 @@
#pragma once
#include <optional>
+#include <stdexcept>
#include <string>
#include <string_view>
-#include <stdexcept>
#include "magic_enum.hpp"
@@ -34,10 +34,9 @@ T enumCast(std::string_view str, bool case_insensitive = false) {
} else {
enum_optional_value = magic_enum::enum_cast<T>(str);
}
- if (enum_optional_value) {
- return enum_optional_value.value();
- }
- throw std::runtime_error("Cannot convert \"" + std::string(str) + "\" to enum class value of enum type \"" + std::string(magic_enum::enum_type_name<T>()) + "\"");
+ if (enum_optional_value) { return enum_optional_value.value(); }
+ throw std::runtime_error("Cannot convert \"" + std::string(str) + "\" to enum class value of enum type \"" +
+ std::string(magic_enum::enum_type_name<T>()) + "\"");
}
} // namespace org::apache::nifi::minifi::utils
diff --git a/core-framework/include/utils/Environment.h b/core-framework/common/include/utils/Environment.h
similarity index 100%
rename from core-framework/include/utils/Environment.h
rename to core-framework/common/include/utils/Environment.h
diff --git a/core-framework/include/utils/FailurePolicy.h b/core-framework/common/include/utils/FailurePolicy.h
similarity index 100%
rename from core-framework/include/utils/FailurePolicy.h
rename to core-framework/common/include/utils/FailurePolicy.h
diff --git a/core-framework/include/utils/GeneralUtils.h b/core-framework/common/include/utils/GeneralUtils.h
similarity index 100%
rename from core-framework/include/utils/GeneralUtils.h
rename to core-framework/common/include/utils/GeneralUtils.h
diff --git a/core-framework/include/utils/Hash.h b/core-framework/common/include/utils/Hash.h
similarity index 100%
rename from core-framework/include/utils/Hash.h
rename to core-framework/common/include/utils/Hash.h
diff --git a/core-framework/common/include/utils/IdHash.h b/core-framework/common/include/utils/IdHash.h
new file mode 100644
index 000000000..645b13858
--- /dev/null
+++ b/core-framework/common/include/utils/IdHash.h
@@ -0,0 +1,40 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "minifi-cpp/utils/Id.h"
+#include "utils/Hash.h"
+
+namespace std {
+template<>
+struct hash<org::apache::nifi::minifi::utils::Identifier> {
+ size_t operator()(const org::apache::nifi::minifi::utils::Identifier& id) const noexcept {
+ static_assert(sizeof(org::apache::nifi::minifi::utils::Identifier) % sizeof(size_t) == 0);
+ constexpr int slices = sizeof(org::apache::nifi::minifi::utils::Identifier) / sizeof(size_t);
+ const auto get_slice = [](const org::apache::nifi::minifi::utils::Identifier& id, size_t idx) -> size_t {
+ size_t result{};
+ memcpy(&result, reinterpret_cast<const unsigned char*>(&id.data_) + idx * sizeof(size_t), sizeof(size_t));
+ return result;
+ };
+ size_t hash = get_slice(id, 0);
+ for (size_t i = 1; i < slices; ++i) {
+ hash = org::apache::nifi::minifi::utils::hash_combine(hash, get_slice(id, i));
+ }
+ return hash;
+ }
+};
+} // namespace std
diff --git a/core-framework/include/utils/OptionalUtils.h b/core-framework/common/include/utils/OptionalUtils.h
similarity index 100%
rename from core-framework/include/utils/OptionalUtils.h
rename to core-framework/common/include/utils/OptionalUtils.h
diff --git a/core-framework/include/utils/ParsingErrors.h b/core-framework/common/include/utils/ParsingErrors.h
similarity index 100%
rename from core-framework/include/utils/ParsingErrors.h
rename to core-framework/common/include/utils/ParsingErrors.h
diff --git a/core-framework/include/utils/ParsingUtils.h b/core-framework/common/include/utils/ParsingUtils.h
similarity index 100%
rename from core-framework/include/utils/ParsingUtils.h
rename to core-framework/common/include/utils/ParsingUtils.h
diff --git a/core-framework/include/utils/PropertyErrors.h b/core-framework/common/include/utils/PropertyErrors.h
similarity index 100%
rename from core-framework/include/utils/PropertyErrors.h
rename to core-framework/common/include/utils/PropertyErrors.h
diff --git a/core-framework/include/utils/SmallString.h b/core-framework/common/include/utils/SmallString.h
similarity index 100%
rename from core-framework/include/utils/SmallString.h
rename to core-framework/common/include/utils/SmallString.h
diff --git a/core-framework/include/utils/StringUtils.h b/core-framework/common/include/utils/StringUtils.h
similarity index 99%
rename from core-framework/include/utils/StringUtils.h
rename to core-framework/common/include/utils/StringUtils.h
index 30b2a1495..be9091a91 100644
--- a/core-framework/include/utils/StringUtils.h
+++ b/core-framework/common/include/utils/StringUtils.h
@@ -452,6 +452,7 @@ nonstd::expected<T, std::error_code> parseNumber(std::string_view input) {
std::string partAfterLastOccurrenceOf(std::string_view input, char delimiter);
+std::string snakeCaseToPascalCase(std::string_view input);
template<typename ... Bases>
struct overload : Bases ... {
diff --git a/core-framework/include/utils/TimeUtil.h b/core-framework/common/include/utils/TimeUtil.h
similarity index 96%
rename from core-framework/include/utils/TimeUtil.h
rename to core-framework/common/include/utils/TimeUtil.h
index c9b1d4534..596715f5a 100644
--- a/core-framework/include/utils/TimeUtil.h
+++ b/core-framework/common/include/utils/TimeUtil.h
@@ -33,7 +33,6 @@
#include <memory>
#include "StringUtils.h"
-#include "minifi-cpp/utils/TimeUtil.h"
// libc++ doesn't define operator<=> on durations, and apparently the operator rewrite rules don't automagically make one
#if defined(_LIBCPP_VERSION)
@@ -249,10 +248,4 @@ inline date::local_seconds roundToNextSecond(date::local_seconds tp) {
return std::chrono::floor<std::chrono::seconds>(tp) + std::chrono::seconds(1);
}
-#ifdef WIN32
-// The tzdata location is set as a global variable in date-tz library
-// We need to set it from from libminifi to effect calls made from libminifi (on Windows)
-void dateSetInstall(const std::string& install);
-#endif
-
} // namespace org::apache::nifi::minifi::utils::timeutils
diff --git a/core-framework/common/include/utils/UnicodeConversion.h b/core-framework/common/include/utils/UnicodeConversion.h
new file mode 100644
index 000000000..db925118f
--- /dev/null
+++ b/core-framework/common/include/utils/UnicodeConversion.h
@@ -0,0 +1,37 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#pragma once
+
+#include <atlbase.h>
+#include <atlconv.h>
+#include <string>
+
+namespace org::apache::nifi::minifi::utils {
+
+inline std::string to_string(const std::wstring& utf16_string) {
+ ATL::CW2A utf8_string(utf16_string.c_str(), CP_UTF8);
+ return {LPSTR{utf8_string}};
+}
+
+inline std::wstring to_wstring(const std::string& utf8_string) {
+ ATL::CA2W utf16_string(utf8_string.c_str(), CP_UTF8);
+ return {LPWSTR{utf16_string}};
+}
+
+} // namespace org::apache::nifi::minifi::utils
diff --git a/core-framework/include/utils/detail/MonadicOperationWrappers.h b/core-framework/common/include/utils/detail/MonadicOperationWrappers.h
similarity index 100%
rename from core-framework/include/utils/detail/MonadicOperationWrappers.h
rename to core-framework/common/include/utils/detail/MonadicOperationWrappers.h
diff --git a/core-framework/include/utils/expected.h b/core-framework/common/include/utils/expected.h
similarity index 100%
rename from core-framework/include/utils/expected.h
rename to core-framework/common/include/utils/expected.h
diff --git a/core-framework/include/utils/meta/detected.h b/core-framework/common/include/utils/meta/detected.h
similarity index 100%
rename from core-framework/include/utils/meta/detected.h
rename to core-framework/common/include/utils/meta/detected.h
diff --git a/core-framework/include/utils/span.h b/core-framework/common/include/utils/span.h
similarity index 100%
rename from core-framework/include/utils/span.h
rename to core-framework/common/include/utils/span.h
diff --git a/core-framework/src/io/InputStream.cpp b/core-framework/common/src/io/InputStream.cpp
similarity index 100%
rename from core-framework/src/io/InputStream.cpp
rename to core-framework/common/src/io/InputStream.cpp
diff --git a/core-framework/src/io/OutputStream.cpp b/core-framework/common/src/io/OutputStream.cpp
similarity index 100%
rename from core-framework/src/io/OutputStream.cpp
rename to core-framework/common/src/io/OutputStream.cpp
diff --git a/core-framework/src/utils/Environment.cpp b/core-framework/common/src/utils/Environment.cpp
similarity index 100%
rename from core-framework/src/utils/Environment.cpp
rename to core-framework/common/src/utils/Environment.cpp
diff --git a/core-framework/common/src/utils/Id.cpp b/core-framework/common/src/utils/Id.cpp
new file mode 100644
index 000000000..d401d0a5e
--- /dev/null
+++ b/core-framework/common/src/utils/Id.cpp
@@ -0,0 +1,133 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "minifi-cpp/utils/Id.h"
+
+#include "utils/StringUtils.h"
+#include "minifi-cpp/utils/gsl.h"
+
+namespace org::apache::nifi::minifi::utils {
+
+Identifier::Identifier(const Data& data) : data_(data) {}
+
+Identifier& Identifier::operator=(const Data& data) {
+ data_ = data;
+ return *this;
+}
+
+Identifier& Identifier::operator=(const std::string& idStr) {
+ const auto id = Identifier::parse(idStr);
+ if (!id) {
+ throw std::runtime_error("Couldn't parse UUID");
+ }
+ *this = id.value();
+ return *this;
+}
+
+bool Identifier::isNil() const {
+ return *this == Identifier{};
+}
+
+bool Identifier::operator!=(const Identifier& other) const {
+ return !(*this == other);
+}
+
+bool Identifier::operator==(const Identifier& other) const {
+ return data_ == other.data_;
+}
+
+bool Identifier::operator<(const Identifier &other) const {
+ return data_ < other.data_;
+}
+
+SmallString<36> Identifier::to_string() const {
+ SmallString<36> uuidStr;
+ // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 long: 16 bytes * 2 hex digits / byte + 4 hyphens
+ int byteIdx = 0;
+ int charIdx = 0;
+
+ // [xxxxxxxx]-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ while (byteIdx < 4) {
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+ }
+ // xxxxxxxx[-]xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ uuidStr[charIdx++] = '-';
+
+ // xxxxxxxx-[xxxx-xxxx-xxxx-]xxxxxxxxxxxx - 3x 2 bytes and a hyphen
+ for (int idx = 0; idx < 3; ++idx) {
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+ uuidStr[charIdx++] = '-';
+ }
+
+ // xxxxxxxx-xxxx-xxxx-xxxx-[xxxxxxxxxxxx] - the rest, i.e. until byte 16
+ while (byteIdx < 16) {
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+ uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+ }
+
+ // null terminator
+ uuidStr[charIdx] = 0;
+ return uuidStr;
+}
+
+std::optional<Identifier> Identifier::parse(const std::string &str) {
+ Identifier id;
+ // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 long: 16 bytes * 2 hex digits / byte + 4 hyphens
+ if (str.length() != 36) return {};
+ int charIdx = 0;
+ int byteIdx = 0;
+ auto input = reinterpret_cast<const uint8_t*>(str.c_str());
+
+ // [xxxxxxxx]-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ while (byteIdx < 4) {
+ if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+ }
+ // xxxxxxxx[-]xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ if (input[charIdx++] != '-') return {};
+
+ // xxxxxxxx-[xxxx-xxxx-xxxx-]xxxxxxxxxxxx - 3x 2 bytes and a hyphen
+ for (size_t idx = 0; idx < 3; ++idx) {
+ if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+ if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+ if (input[charIdx++] != '-') return {};
+ }
+
+ // xxxxxxxx-xxxx-xxxx-xxxx-[xxxxxxxxxxxx] - the rest, i.e. until byte 16
+ while (byteIdx < 16) {
+ if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+ }
+ return id;
+}
+
+bool Identifier::parseByte(Data &data, const uint8_t *input, int &charIdx, int &byteIdx) {
+ uint8_t upper = 0;
+ uint8_t lower = 0;
+ if (!string::from_hex(input[charIdx], upper)
+ || !string::from_hex(input[charIdx + 1], lower)) {
+ return false;
+ }
+ charIdx += 2;
+ data[byteIdx++] = (upper << 4) | lower;
+ return true;
+}
+
+} // namespace org::apache::nifi::minifi::utils
diff --git a/core-framework/src/utils/ParsingUtils.cpp b/core-framework/common/src/utils/ParsingUtils.cpp
similarity index 100%
rename from core-framework/src/utils/ParsingUtils.cpp
rename to core-framework/common/src/utils/ParsingUtils.cpp
diff --git a/core-framework/src/utils/StringUtils.cpp b/core-framework/common/src/utils/StringUtils.cpp
similarity index 97%
rename from core-framework/src/utils/StringUtils.cpp
rename to core-framework/common/src/utils/StringUtils.cpp
index 5ead1562b..eee74742d 100644
--- a/core-framework/src/utils/StringUtils.cpp
+++ b/core-framework/common/src/utils/StringUtils.cpp
@@ -610,4 +610,20 @@ std::string partAfterLastOccurrenceOf(std::string_view input, char delimiter) {
return std::string{input.substr(last_pos + 1)};
}
+std::string snakeCaseToPascalCase(std::string_view input) {
+ std::string result;
+ bool should_uppercase_next_letter = true;
+ for (char ch : input) {
+ if (ch == '_') {
+ should_uppercase_next_letter = true;
+ } else if (should_uppercase_next_letter) {
+ result += static_cast<char>(std::toupper(static_cast<unsigned char>(ch)));
+ should_uppercase_next_letter = false;
+ } else {
+ result += static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
+ }
+ }
+ return result;
+}
+
} // namespace org::apache::nifi::minifi::utils::string
diff --git a/core-framework/src/utils/TimeUtil.cpp b/core-framework/common/src/utils/TimeUtil.cpp
similarity index 91%
rename from core-framework/src/utils/TimeUtil.cpp
rename to core-framework/common/src/utils/TimeUtil.cpp
index 67b5ecc6e..5514b5bec 100644
--- a/core-framework/src/utils/TimeUtil.cpp
+++ b/core-framework/common/src/utils/TimeUtil.cpp
@@ -114,13 +114,4 @@ std::optional<std::chrono::system_clock::time_point> parseRfc3339(const std::str
return date::sys_days(date_part) + time_part - offset;
}
-#ifdef WIN32
-// If minifi is not installed through the MSI installer, then TZDATA might be missing
-// date::set_install can point to the TZDATA location, but it has to be called from each library/executable that wants to use timezones
-void dateSetInstall(const std::string& install) {
- date::set_install(install);
- dateSetGlobalInstall(install);
-}
-#endif
-
} // namespace org::apache::nifi::minifi::utils::timeutils
diff --git a/core-framework/include/agent/agent_docs.h b/core-framework/include/agent/agent_docs.h
index 09142be4e..71149aca9 100644
--- a/core-framework/include/agent/agent_docs.h
+++ b/core-framework/include/agent/agent_docs.h
@@ -25,6 +25,9 @@
#include "utils/StringUtils.h"
#include "core/ClassName.h"
+#include "minifi-cpp/core/OutputAttribute.h"
+#include "minifi-cpp/core/ControllerServiceApi.h"
+#include "minifi-cpp/core/DynamicProperty.h"
namespace org::apache::nifi::minifi {
@@ -37,6 +40,18 @@ inline auto toVector(std::span<const core::RelationshipDefinition> relationships
return std::vector<core::Relationship>(relationships.begin(), relationships.end());
}
+inline auto toVector(std::span<const core::OutputAttributeReference> attributes) {
+ return std::vector<core::OutputAttribute>(attributes.begin(), attributes.end());
+}
+
+inline auto toVector(std::span<const core::ControllerServiceApiDefinition> apis) {
+ return std::vector<core::ControllerServiceApi>(apis.begin(), apis.end());
+}
+
+inline auto toVector(std::span<const core::DynamicPropertyDefinition> properties) {
+ return std::vector<core::DynamicProperty>(properties.begin(), properties.end());
+}
+
template<typename T>
std::string classNameWithDots() {
std::string class_name{core::className<T>()};
@@ -55,9 +70,9 @@ void AgentDocs::createClassDescription(const std::string& group, const std::stri
.full_name_ = detail::classNameWithDots<Class>(),
.description_ = Class::Description,
.class_properties_ = detail::toVector(Class::Properties),
- .dynamic_properties_ = Class::DynamicProperties,
+ .dynamic_properties_ = detail::toVector(Class::DynamicProperties),
.class_relationships_ = detail::toVector(Class::Relationships),
- .output_attributes_ = Class::OutputAttributes,
+ .output_attributes_ = detail::toVector(Class::OutputAttributes),
.supports_dynamic_properties_ = Class::SupportsDynamicProperties,
.supports_dynamic_relationships_ = Class::SupportsDynamicRelationships,
.inputRequirement_ = toString(Class::InputRequirement),
@@ -70,7 +85,7 @@ void AgentDocs::createClassDescription(const std::string& group, const std::stri
.full_name_ = detail::classNameWithDots<Class>(),
.description_ = Class::Description,
.class_properties_ = detail::toVector(Class::Properties),
- .api_implementations = Class::ImplementsApis,
+ .api_implementations = detail::toVector(Class::ImplementsApis),
.supports_dynamic_properties_ = Class::SupportsDynamicProperties
});
} else if constexpr (Type == ResourceType::InternalResource) {
diff --git a/core-framework/include/core/ConfigurableComponentImpl.h b/core-framework/include/core/ConfigurableComponentImpl.h
index 0335e8043..8b212824e 100644
--- a/core-framework/include/core/ConfigurableComponentImpl.h
+++ b/core-framework/include/core/ConfigurableComponentImpl.h
@@ -38,6 +38,7 @@ class ConfigurableComponentImpl : public virtual ConfigurableComponent {
nonstd::expected<void, std::error_code> setDynamicProperty(std::string name, std::string value) override;
void setSupportedProperties(std::span<const PropertyReference> properties);
+ void setSupportedProperties(std::span<const Property> properties);
[[nodiscard]] std::map<std::string, Property, std::less<>> getSupportedProperties() const override;
[[nodiscard]] nonstd::expected<Property, std::error_code> getSupportedProperty(std::string_view name) const override;
diff --git a/core-framework/include/core/ObjectFactory.h b/core-framework/include/core/ObjectFactory.h
index e321a6a1e..dda11ab96 100644
--- a/core-framework/include/core/ObjectFactory.h
+++ b/core-framework/include/core/ObjectFactory.h
@@ -20,7 +20,7 @@
#include <string>
#include <memory>
#include <utility>
-#include "ClassName.h"
+#include "core/ClassName.h"
#include "minifi-cpp/core/ObjectFactory.h"
namespace org::apache::nifi::minifi::core {
diff --git a/core-framework/include/core/ProcessorFactoryImpl.h b/core-framework/include/core/ProcessorFactoryImpl.h
index d4ea016c8..e7583bcf1 100644
--- a/core-framework/include/core/ProcessorFactoryImpl.h
+++ b/core-framework/include/core/ProcessorFactoryImpl.h
@@ -20,7 +20,7 @@
#include <string>
#include <memory>
#include <utility>
-#include "ClassName.h"
+#include "core/ClassName.h"
#include "minifi-cpp/core/ProcessorFactory.h"
namespace org::apache::nifi::minifi::core {
diff --git a/core-framework/include/core/ProcessorImpl.h b/core-framework/include/core/ProcessorImpl.h
index cf409012a..aa4c34006 100644
--- a/core-framework/include/core/ProcessorImpl.h
+++ b/core-framework/include/core/ProcessorImpl.h
@@ -31,7 +31,7 @@
#include "minifi-cpp/core/Property.h"
#include "core/Core.h"
#include "minifi-cpp/core/Annotation.h"
-#include "minifi-cpp/core/DynamicProperty.h"
+#include "minifi-cpp/core/DynamicPropertyDefinition.h"
#include "minifi-cpp/core/Scheduling.h"
#include "minifi-cpp/core/ProcessorMetricsExtension.h"
#include "minifi-cpp/utils/gsl.h"
@@ -100,6 +100,7 @@ class ProcessorImpl : public virtual ProcessorApi {
void setSupportedRelationships(std::span<const RelationshipDefinition> relationships);
void setSupportedProperties(std::span<const PropertyReference> properties);
+ void setSupportedProperties(std::span<const Property> properties);
virtual void initialize() {}
@@ -121,13 +122,13 @@ class ProcessorImpl : public virtual ProcessorApi {
return metrics_extension_;
}
- static constexpr auto DynamicProperties = std::array<DynamicProperty, 0>{};
+ static constexpr auto DynamicProperties = std::array<DynamicPropertyDefinition, 0>{};
static constexpr auto OutputAttributes = std::array<OutputAttributeReference, 0>{};
void restore(const std::shared_ptr<FlowFile>& file) override;
- void setLoggerCallback(const std::function<void(logging::LOG_LEVEL level, const std::string& message)>& /*callback*/) override;
+ void forEachLogger(const std::function<void(std::shared_ptr<logging::Logger>)>& callback) override;
std::string getName() const;
utils::Identifier getUUID() const;
diff --git a/core-framework/include/io/BaseStream.h b/core-framework/include/io/BaseStream.h
index 99b62d159..899df1ba8 100644
--- a/core-framework/include/io/BaseStream.h
+++ b/core-framework/include/io/BaseStream.h
@@ -18,8 +18,8 @@
#pragma once
-#include "InputStream.h"
-#include "OutputStream.h"
+#include "io/InputStream.h"
+#include "io/OutputStream.h"
#include "minifi-cpp/io/BaseStream.h"
namespace org::apache::nifi::minifi::io {
diff --git a/core-framework/include/io/CRCStream.h b/core-framework/include/io/CRCStream.h
index a53e44c5d..730847df4 100644
--- a/core-framework/include/io/CRCStream.h
+++ b/core-framework/include/io/CRCStream.h
@@ -32,8 +32,8 @@
#include <arpa/inet.h>
#endif
-#include "InputStream.h"
-#include "OutputStream.h"
+#include "io/InputStream.h"
+#include "io/OutputStream.h"
#include "minifi-cpp/Exception.h"
namespace org::apache::nifi::minifi::io {
diff --git a/core-framework/include/io/StreamSlice.h b/core-framework/include/io/StreamSlice.h
index f11a2aba4..bc8d98ad9 100644
--- a/core-framework/include/io/StreamSlice.h
+++ b/core-framework/include/io/StreamSlice.h
@@ -21,7 +21,7 @@
#include <memory>
#include "minifi-cpp/io/StreamCallback.h"
-#include "InputStream.h"
+#include "io/InputStream.h"
namespace org::apache::nifi::minifi::io {
diff --git a/core-framework/include/io/ZlibStream.h b/core-framework/include/io/ZlibStream.h
index b90c76add..ee5fe70d9 100644
--- a/core-framework/include/io/ZlibStream.h
+++ b/core-framework/include/io/ZlibStream.h
@@ -24,8 +24,8 @@
#include <memory>
#include <vector>
-#include "OutputStream.h"
-#include "Stream.h"
+#include "io/OutputStream.h"
+#include "io/Stream.h"
#include "minifi-cpp/core/logging/Logger.h"
#include "minifi-cpp/utils/gsl.h"
diff --git a/core-framework/include/utils/Id.h b/core-framework/include/utils/Id.h
index b9f7208d2..d289ac288 100644
--- a/core-framework/include/utils/Id.h
+++ b/core-framework/include/utils/Id.h
@@ -33,7 +33,7 @@ class uuid;
#include "minifi-cpp/core/logging/Logger.h"
#include "minifi-cpp/properties/Properties.h"
#include "utils/SmallString.h"
-#include "Hash.h"
+#include "utils/IdHash.h"
#define UUID_TIME_IMPL 0
#define UUID_RANDOM_IMPL 1
@@ -94,23 +94,3 @@ class NonRepeatingStringGenerator {
};
} // namespace org::apache::nifi::minifi::utils
-
-namespace std {
-template<>
-struct hash<org::apache::nifi::minifi::utils::Identifier> {
- size_t operator()(const org::apache::nifi::minifi::utils::Identifier& id) const noexcept {
- static_assert(sizeof(org::apache::nifi::minifi::utils::Identifier) % sizeof(size_t) == 0);
- constexpr int slices = sizeof(org::apache::nifi::minifi::utils::Identifier) / sizeof(size_t);
- const auto get_slice = [](const org::apache::nifi::minifi::utils::Identifier& id, size_t idx) -> size_t {
- size_t result{};
- memcpy(&result, reinterpret_cast<const unsigned char*>(&id.data_) + idx * sizeof(size_t), sizeof(size_t));
- return result;
- };
- size_t hash = get_slice(id, 0);
- for (size_t i = 1; i < slices; ++i) {
- hash = org::apache::nifi::minifi::utils::hash_combine(hash, get_slice(id, i));
- }
- return hash;
- }
-};
-} // namespace std
diff --git a/core-framework/include/utils/Locations.h b/core-framework/include/utils/Locations.h
index 0083acdd5..0f4637c84 100644
--- a/core-framework/include/utils/Locations.h
+++ b/core-framework/include/utils/Locations.h
@@ -19,7 +19,7 @@
#include <filesystem>
#include "Defaults.h"
-#include "Environment.h"
+#include "utils/Environment.h"
#include "file/FileUtils.h"
namespace org::apache::nifi::minifi::utils {
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/core-framework/include/utils/TimeZoneUtils.h
similarity index 70%
copy from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
copy to core-framework/include/utils/TimeZoneUtils.h
index 909ba731d..54c4317dc 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
+++ b/core-framework/include/utils/TimeZoneUtils.h
@@ -16,17 +16,14 @@
*/
#pragma once
-#include <memory>
#include <string>
-#include "minifi-cpp/utils/Id.h"
-#include "minifi-cpp/core/logging/Logger.h"
-namespace org::apache::nifi::minifi::core {
+namespace org::apache::nifi::minifi::utils::timeutils {
-struct ProcessorMetadata {
- utils::Identifier uuid;
- std::string name;
- std::shared_ptr<logging::Logger> logger;
-};
+#ifdef WIN32
+// The tzdata location is set as a global variable in date-tz library
+// We need to set it from from libminifi to effect calls made from libminifi (on Windows)
+void dateSetInstall(const std::string& install);
+#endif
-} // namespace org::apache::nifi::minifi::core
+} // namespace org::apache::nifi::minifi::utils::timeutils
diff --git a/core-framework/include/utils/file/FileUtils.h b/core-framework/include/utils/file/FileUtils.h
index 36f6d5e53..06b1cc2bd 100644
--- a/core-framework/include/utils/file/FileUtils.h
+++ b/core-framework/include/utils/file/FileUtils.h
@@ -433,6 +433,15 @@ inline std::string get_content(const std::filesystem::path& file_name) {
bool contains(const std::filesystem::path& file_path, std::string_view text_to_search);
+// finds a string of length at most @max_length with a given @prefix
+std::optional<std::string> findSubstringWithPrefix(std::istream& stream, std::string_view prefix, size_t max_length);
+
+// finds a string of length at most @max_length with a given @prefix
+inline std::optional<std::string> findSubstringWithPrefix(const std::filesystem::path& file_path, std::string_view prefix, size_t max_length) {
+ gsl_ExpectsAudit(std::filesystem::exists(file_path));
+ std::ifstream ifs{file_path, std::ios::binary};
+ return findSubstringWithPrefix(ifs, prefix, max_length);
+}
inline std::optional<std::string> get_file_owner(const std::filesystem::path& file_path) {
#ifndef WIN32
diff --git a/core-framework/include/utils/net/NetworkInterfaceInfo.h b/core-framework/include/utils/net/NetworkInterfaceInfo.h
index 64fb2f212..bb7b9c97b 100644
--- a/core-framework/include/utils/net/NetworkInterfaceInfo.h
+++ b/core-framework/include/utils/net/NetworkInterfaceInfo.h
@@ -20,7 +20,7 @@
#include <string>
#include <vector>
#include <memory>
-#include <unordered_map>
+#include <functional>
#include <optional>
#include "minifi-cpp/core/logging/Logger.h"
diff --git a/core-framework/src/core/ConfigurableComponentImpl.cpp b/core-framework/src/core/ConfigurableComponentImpl.cpp
index 101b2403a..db0edab2f 100644
--- a/core-framework/src/core/ConfigurableComponentImpl.cpp
+++ b/core-framework/src/core/ConfigurableComponentImpl.cpp
@@ -31,6 +31,15 @@ void ConfigurableComponentImpl::setSupportedProperties(std::span<const core::Pro
for (const auto& item: properties) { supported_properties_.emplace(item.name, item); }
}
+void ConfigurableComponentImpl::setSupportedProperties(std::span<const Property> properties) {
+ if (!canEdit()) { return; }
+
+ std::lock_guard<std::mutex> lock(configuration_mutex_);
+ supported_properties_.clear();
+ for (const auto& item: properties) { supported_properties_.emplace(item.getName(), item); }
+}
+
+
nonstd::expected<std::string, std::error_code> ConfigurableComponentImpl::getProperty(const std::string_view name) const {
const std::lock_guard<std::mutex> lock(configuration_mutex_);
const auto it = supported_properties_.find(name);
diff --git a/core-framework/src/core/ProcessorImpl.cpp b/core-framework/src/core/ProcessorImpl.cpp
index c45c0ff8f..31ec58e8b 100644
--- a/core-framework/src/core/ProcessorImpl.cpp
+++ b/core-framework/src/core/ProcessorImpl.cpp
@@ -58,7 +58,7 @@ bool ProcessorImpl::isWorkAvailable() {
}
void ProcessorImpl::restore(const std::shared_ptr<FlowFile>& /*file*/) {
- gsl_Assert("Not implemented");
+ gsl_Assert(false && "Not implemented");
}
std::string ProcessorImpl::getName() const {
@@ -90,8 +90,13 @@ void ProcessorImpl::setSupportedProperties(std::span<const PropertyReference> pr
descriptor_->setSupportedProperties(properties);
}
-void ProcessorImpl::setLoggerCallback(const std::function<void(logging::LOG_LEVEL level, const std::string& message)>& callback) {
- logger_->setLogCallback(callback);
+void ProcessorImpl::setSupportedProperties(std::span<const Property> properties) {
+ gsl_Expects(descriptor_);
+ descriptor_->setSupportedProperties(properties);
+}
+
+void ProcessorImpl::forEachLogger(const std::function<void(std::shared_ptr<logging::Logger>)>& callback) {
+ callback(logger_);
}
} // namespace org::apache::nifi::minifi::core
diff --git a/core-framework/src/utils/Id.cpp b/core-framework/src/utils/Id.cpp
index 3037827f1..c5267eca7 100644
--- a/core-framework/src/utils/Id.cpp
+++ b/core-framework/src/utils/Id.cpp
@@ -72,112 +72,6 @@ namespace {
} // namespace
#endif
-Identifier::Identifier(const Data& data) : data_(data) {}
-
-Identifier& Identifier::operator=(const Data& data) {
- data_ = data;
- return *this;
-}
-
-Identifier& Identifier::operator=(const std::string& idStr) {
- const auto id = Identifier::parse(idStr);
- if (!id) {
- throw std::runtime_error("Couldn't parse UUID");
- }
- *this = id.value();
- return *this;
-}
-
-bool Identifier::isNil() const {
- return *this == Identifier{};
-}
-
-bool Identifier::operator!=(const Identifier& other) const {
- return !(*this == other);
-}
-
-bool Identifier::operator==(const Identifier& other) const {
- return data_ == other.data_;
-}
-
-bool Identifier::operator<(const Identifier &other) const {
- return data_ < other.data_;
-}
-
-SmallString<36> Identifier::to_string() const {
- SmallString<36> uuidStr;
- // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 long: 16 bytes * 2 hex digits / byte + 4 hyphens
- int byteIdx = 0;
- int charIdx = 0;
-
- // [xxxxxxxx]-xxxx-xxxx-xxxx-xxxxxxxxxxxx
- while (byteIdx < 4) {
- uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
- uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
- }
- // xxxxxxxx[-]xxxx-xxxx-xxxx-xxxxxxxxxxxx
- uuidStr[charIdx++] = '-';
-
- // xxxxxxxx-[xxxx-xxxx-xxxx-]xxxxxxxxxxxx - 3x 2 bytes and a hyphen
- for (int idx = 0; idx < 3; ++idx) {
- uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
- uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
- uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
- uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
- uuidStr[charIdx++] = '-';
- }
-
- // xxxxxxxx-xxxx-xxxx-xxxx-[xxxxxxxxxxxx] - the rest, i.e. until byte 16
- while (byteIdx < 16) {
- uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
- uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
- }
-
- // null terminator
- uuidStr[charIdx] = 0;
- return uuidStr;
-}
-
-std::optional<Identifier> Identifier::parse(const std::string &str) {
- Identifier id;
- // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 long: 16 bytes * 2 hex digits / byte + 4 hyphens
- if (str.length() != 36) return {};
- int charIdx = 0;
- int byteIdx = 0;
- auto input = reinterpret_cast<const uint8_t*>(str.c_str());
-
- // [xxxxxxxx]-xxxx-xxxx-xxxx-xxxxxxxxxxxx
- while (byteIdx < 4) {
- if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
- }
- // xxxxxxxx[-]xxxx-xxxx-xxxx-xxxxxxxxxxxx
- if (input[charIdx++] != '-') return {};
-
- // xxxxxxxx-[xxxx-xxxx-xxxx-]xxxxxxxxxxxx - 3x 2 bytes and a hyphen
- for (size_t idx = 0; idx < 3; ++idx) {
- if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
- if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
- if (input[charIdx++] != '-') return {};
- }
-
- // xxxxxxxx-xxxx-xxxx-xxxx-[xxxxxxxxxxxx] - the rest, i.e. until byte 16
- while (byteIdx < 16) {
- if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
- }
- return id;
-}
-
-bool Identifier::parseByte(Data &data, const uint8_t *input, int &charIdx, int &byteIdx) {
- uint8_t upper = 0;
- uint8_t lower = 0;
- if (!string::from_hex(input[charIdx++], upper) // NOLINT(bugprone-inc-dec-in-conditions)
- || !string::from_hex(input[charIdx++], lower)) { // NOLINT(bugprone-inc-dec-in-conditions)
- return false;
- }
- data[byteIdx++] = (upper << 4) | lower;
- return true;
-}
-
IdGenerator::IdGenerator()
: implementation_(UUID_TIME_IMPL),
logger_(core::logging::LoggerFactory<IdGenerator>::getLogger()),
diff --git a/core-framework/include/utils/UnicodeConversion.h b/core-framework/src/utils/TimeZoneUtils.cpp
similarity index 58%
rename from core-framework/include/utils/UnicodeConversion.h
rename to core-framework/src/utils/TimeZoneUtils.cpp
index dfd0e26ef..0fe675e81 100644
--- a/core-framework/include/utils/UnicodeConversion.h
+++ b/core-framework/src/utils/TimeZoneUtils.cpp
@@ -1,7 +1,4 @@
/**
- * @file UnicodeConversion.h
- * Unicode conversion functions
- *
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -18,22 +15,23 @@
* limitations under the License.
*/
-#pragma once
+#include "utils/TimeZoneUtils.h"
-#include <atlbase.h>
-#include <atlconv.h>
-#include <string>
+#include "minifi-cpp/utils/TimeUtil.h"
-namespace org::apache::nifi::minifi::utils {
+#ifdef WIN32
+#include "date/tz.h"
+#endif
-inline std::string to_string(const std::wstring& utf16_string) {
- ATL::CW2A utf8_string(utf16_string.c_str(), CP_UTF8);
- return {LPSTR{utf8_string}};
-}
+namespace org::apache::nifi::minifi::utils::timeutils {
-inline std::wstring to_wstring(const std::string& utf8_string) {
- ATL::CA2W utf16_string(utf8_string.c_str(), CP_UTF8);
- return {LPWSTR{utf16_string}};
+#ifdef WIN32
+// If minifi is not installed through the MSI installer, then TZDATA might be missing
+// date::set_install can point to the TZDATA location, but it has to be called from each library/executable that wants to use timezones
+void dateSetInstall(const std::string& install) {
+ date::set_install(install);
+ dateSetGlobalInstall(install);
}
+#endif
-} // namespace org::apache::nifi::minifi::utils
+} // namespace org::apache::nifi::minifi::utils::timeutils
diff --git a/core-framework/src/utils/file/FileUtils.cpp b/core-framework/src/utils/file/FileUtils.cpp
index e24b903e1..1e8dc61ff 100644
--- a/core-framework/src/utils/file/FileUtils.cpp
+++ b/core-framework/src/utils/file/FileUtils.cpp
@@ -71,6 +71,31 @@ bool contains(const std::filesystem::path& file_path, std::string_view text_to_s
return std::search(view.begin(), view.end(), searcher) != view.end();
}
+// finds a string of length at most @max_length with a given @prefix
+std::optional<std::string> findSubstringWithPrefix(std::istream& stream, std::string_view prefix, size_t max_length) {
+ gsl_Expects(prefix.size() <= max_length);
+ gsl_Expects(max_length <= 8_KiB);
+ std::array<char, 16_KiB> buf{};
+ std::span<char> view;
+
+ Searcher searcher(prefix.begin(), prefix.end());
+
+ do {
+ std::copy(buf.end() - max_length, buf.end(), buf.begin());
+ stream.read(buf.data() + max_length, gsl::narrow<std::streamsize>(buf.size() - max_length));
+ view = std::span<char>(buf.data(), max_length + gsl::narrow<size_t>(stream.gcount()));
+ auto prefix_end = view.begin() + gsl::narrow<std::span<char>::difference_type>(view.size() - (max_length - prefix.size()));
+ if (auto it = std::search(view.begin(), prefix_end, searcher); it != prefix_end) {
+ return std::string{it, it + gsl::narrow<std::span<char>::difference_type>(max_length)};
+ }
+ } while (stream);
+ // there might be a partial (less than max_length) at the end
+ if (auto it = std::search(view.begin(), view.end(), searcher); it != view.end()) {
+ return std::string{it, view.end()};
+ }
+ return std::nullopt;
+}
+
std::chrono::system_clock::time_point to_sys(std::chrono::file_clock::time_point file_time) {
#if defined(WIN32)
// workaround for https://github.com/microsoft/STL/issues/2446
diff --git a/extension-framework/CMakeLists.txt b/extension-framework/CMakeLists.txt
index 8fbbe0ede..9bcfd285c 100644
--- a/extension-framework/CMakeLists.txt
+++ b/extension-framework/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(cpp-extension-lib)
+
file(GLOB SOURCES
src/*.cpp
src/core/*.cpp
diff --git a/extensions/llamacpp/CMakeLists.txt b/extension-framework/cpp-extension-lib/CMakeLists.txt
similarity index 57%
copy from extensions/llamacpp/CMakeLists.txt
copy to extension-framework/cpp-extension-lib/CMakeLists.txt
index e10bea82b..6b22dd980 100644
--- a/extensions/llamacpp/CMakeLists.txt
+++ b/extension-framework/cpp-extension-lib/CMakeLists.txt
@@ -1,5 +1,5 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
+#
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
@@ -17,21 +17,11 @@
# under the License.
#
-if (NOT (ENABLE_ALL OR ENABLE_LLAMACPP))
- return()
-endif()
-
-include(LlamaCpp)
-
-include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
-
-file(GLOB SOURCES "processors/*.cpp")
-
-add_minifi_library(minifi-llamacpp SHARED ${SOURCES})
-target_include_directories(minifi-llamacpp PUBLIC "${CMAKE_SOURCE_DIR}/extensions/llamacpp")
-target_include_directories(minifi-llamacpp PUBLIC "${LLAMACPP_INCLUDE_DIRS}")
+file(GLOB SOURCES src/core/*.cpp src/core/logging/*.cpp src/utils/*.cpp)
-target_link_libraries(minifi-llamacpp ${LIBMINIFI} llama)
+add_minifi_library(minifi-cpp-extension-lib STATIC ${SOURCES})
+target_include_directories(minifi-cpp-extension-lib PUBLIC include)
+target_link_libraries(minifi-cpp-extension-lib PUBLIC minifi-core-framework-common minifi-c-api)
-register_extension(minifi-llamacpp "LLAMACPP EXTENSION" LLAMACPP-EXTENSION "Provides llama.cpp support" "extensions/llamacpp/tests")
+add_subdirectory(libtest)
diff --git a/minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h b/extension-framework/cpp-extension-lib/include/api/core/FlowFile.h
similarity index 67%
copy from minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h
copy to extension-framework/cpp-extension-lib/include/api/core/FlowFile.h
index acab5ce10..833dfdaf7 100644
--- a/minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h
+++ b/extension-framework/cpp-extension-lib/include/api/core/FlowFile.h
@@ -15,20 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
#pragma once
-#include <string_view>
#include <memory>
+#include <stdexcept>
+#include "minifi-c.h"
-#include "Logger.h"
-#include "minifi-cpp/core/Core.h"
-
-namespace org::apache::nifi::minifi::core::logging {
+namespace org::apache::nifi::minifi::api::core {
-class LoggerFactoryBase {
- public:
- static std::shared_ptr<Logger> getAliasedLogger(std::string_view name, const std::optional<utils::Identifier>& id = {});
+struct EnsureMovedFromDeleter {
+ void operator()(MinifiFlowFile* ff) {
+ if (ff) {
+ throw std::logic_error("Each flowfile should be either transferred or removed");
+ }
+ }
};
-} // namespace org::apache::nifi::minifi::core::logging
+using FlowFile = std::unique_ptr<MinifiFlowFile, EnsureMovedFromDeleter>;
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h b/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h
new file mode 100644
index 000000000..59eb52e02
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "minifi-c.h"
+#include "nonstd/expected.hpp"
+#include "minifi-cpp/core/PropertyDefinition.h"
+#include "api/core/FlowFile.h"
+
+namespace org::apache::nifi::minifi::api::core {
+
+class ProcessContext {
+ public:
+ explicit ProcessContext(MinifiProcessContext* impl): impl_(impl) {}
+
+ nonstd::expected<std::string, std::error_code> getProperty(std::string_view name, const FlowFile* flow_file = nullptr) const;
+ nonstd::expected<std::string, std::error_code> getProperty(const minifi::core::PropertyReference& property_reference, const FlowFile* flow_file = nullptr) const {
+ return getProperty(property_reference.name, flow_file);
+ }
+
+ bool hasNonEmptyProperty(std::string_view name) const;
+
+ std::string getProcessorName() const;
+
+ private:
+ MinifiProcessContext* impl_;
+};
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h b/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h
new file mode 100644
index 000000000..e3e812249
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h
@@ -0,0 +1,54 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <span>
+
+#include "minifi-c.h"
+#include "minifi-cpp/core/Relationship.h"
+#include "minifi-cpp/io/StreamCallback.h"
+#include "FlowFile.h"
+
+namespace org::apache::nifi::minifi::api::core {
+
+class ProcessSession {
+ public:
+ explicit ProcessSession(MinifiProcessSession* impl): impl_(impl) {}
+
+ FlowFile create(const FlowFile* parent = nullptr);
+ FlowFile get();
+ void transfer(FlowFile ff, const minifi::core::Relationship& relationship);
+ void remove(FlowFile ff);
+ void write(FlowFile& flow, const io::OutputStreamCallback& callback);
+ void read(FlowFile& flow, const io::InputStreamCallback& callback);
+
+ void setAttribute(FlowFile& ff, std::string_view key, std::string value);
+ void removeAttribute(FlowFile& ff, std::string_view key);
+ std::optional<std::string> getAttribute(FlowFile& ff, std::string_view key);
+ std::map<std::string, std::string> getAttributes(FlowFile& ff);
+
+ void writeBuffer(FlowFile& flow_file, std::span<const char> buffer);
+ void writeBuffer(FlowFile& flow_file, std::span<const std::byte> buffer);
+ std::vector<std::byte> readBuffer(FlowFile& flow_file);
+
+ private:
+ MinifiProcessSession* impl_;
+};
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/extension-framework/cpp-extension-lib/include/api/core/ProcessorImpl.h b/extension-framework/cpp-extension-lib/include/api/core/ProcessorImpl.h
new file mode 100644
index 000000000..fcace11eb
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/include/api/core/ProcessorImpl.h
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <atomic>
+#include <condition_variable>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "minifi-cpp/core/DynamicPropertyDefinition.h"
+#include "minifi-cpp/utils/Id.h"
+#include "minifi-cpp/core/OutputAttributeDefinition.h"
+#include "minifi-cpp/core/ProcessorMetadata.h"
+#include "FlowFile.h"
+#include "PublishedMetrics.h"
+#include "logging/Logger.h"
+#include "utils/SmallString.h"
+
+namespace org::apache::nifi::minifi::api {
+
+class Connection;
+
+namespace core {
+
+class ProcessContext;
+class ProcessSession;
+class ProcessSessionFactory;
+
+class ProcessorImpl {
+ public:
+ explicit ProcessorImpl(minifi::core::ProcessorMetadata metadata);
+
+ ProcessorImpl(const ProcessorImpl&) = delete;
+ ProcessorImpl(ProcessorImpl&&) = delete;
+ ProcessorImpl& operator=(const ProcessorImpl&) = delete;
+ ProcessorImpl& operator=(ProcessorImpl&&) = delete;
+
+ virtual ~ProcessorImpl();
+
+ void setTriggerWhenEmpty(bool trigger_when_empty) {
+ trigger_when_empty_ = trigger_when_empty;
+ }
+
+ virtual bool getTriggerWhenEmpty() const {
+ return trigger_when_empty_;
+ }
+
+ MinifiStatus onTrigger(ProcessContext&, ProcessSession&);
+
+ MinifiStatus onSchedule(ProcessContext&);
+
+ virtual void onUnSchedule() {}
+
+ virtual bool isWorkAvailable();
+
+ static constexpr auto DynamicProperties = std::array<minifi::core::DynamicPropertyDefinition, 0>{};
+
+ static constexpr auto OutputAttributes = std::array<minifi::core::OutputAttributeReference, 0>{};
+
+ virtual void restore(const std::shared_ptr<FlowFile>& file);
+
+ std::string getName() const;
+ utils::Identifier getUUID() const;
+ utils::SmallString<36> getUUIDStr() const;
+
+ virtual PublishedMetrics calculateMetrics() const {return {};}
+
+ protected:
+ virtual MinifiStatus onTriggerImpl(ProcessContext&, ProcessSession&) {return MINIFI_STATUS_SUCCESS;}
+
+ virtual MinifiStatus onScheduleImpl(ProcessContext&) {return MINIFI_STATUS_SUCCESS;}
+
+ minifi::core::ProcessorMetadata metadata_;
+
+ std::atomic<bool> trigger_when_empty_;
+
+ std::shared_ptr<minifi::core::logging::Logger> logger_;
+
+ private:
+ mutable std::mutex mutex_;
+};
+
+} // namespace core
+} // namespace org::apache::nifi::minifi::api
diff --git a/minifi-api/include/minifi-cpp/core/RelationshipDefinition.h b/extension-framework/cpp-extension-lib/include/api/core/PublishedMetrics.h
similarity index 69%
copy from minifi-api/include/minifi-cpp/core/RelationshipDefinition.h
copy to extension-framework/cpp-extension-lib/include/api/core/PublishedMetrics.h
index 29dfdb03d..7ec8147c8 100644
--- a/minifi-api/include/minifi-cpp/core/RelationshipDefinition.h
+++ b/extension-framework/cpp-extension-lib/include/api/core/PublishedMetrics.h
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -16,13 +16,16 @@
*/
#pragma once
-#include <string_view>
+#include <vector>
+#include <string>
-namespace org::apache::nifi::minifi::core {
+namespace org::apache::nifi::minifi::api::core {
-struct RelationshipDefinition {
- std::string_view name;
- std::string_view description;
+struct PublishedMetric {
+ std::string name;
+ double value;
};
-} // namespace org::apache::nifi::minifi::core
+using PublishedMetrics = std::vector<PublishedMetric>;
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/extension-framework/cpp-extension-lib/include/api/core/Resource.h b/extension-framework/cpp-extension-lib/include/api/core/Resource.h
new file mode 100644
index 000000000..f144b95b1
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/include/api/core/Resource.h
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
+#endif
+
+#include "minifi-c.h"
+#include "core/ClassName.h"
+#include "api/utils/minifi-c-utils.h"
+#include "ProcessContext.h"
+#include "ProcessSession.h"
+#include "FlowFile.h"
+#include "minifi-cpp/core/ProcessorMetadata.h"
+#include "logging/Logger.h"
+
+namespace org::apache::nifi::minifi::api::core {
+
+template<typename Class, typename Fn>
+void useProcessorClassDescription(Fn&& fn) {
+ std::vector<std::vector<MinifiStringView>> string_vector_cache;
+
+ const auto full_name = minifi::core::className<Class>();
+
+ std::vector<MinifiPropertyDefinition> class_properties = utils::toProperties(Class::Properties, string_vector_cache);
+ std::vector<MinifiDynamicPropertyDefinition> dynamic_properties;
+ for (auto& prop : Class::DynamicProperties) {
+ dynamic_properties.push_back(MinifiDynamicPropertyDefinition {
+ .name = utils::toStringView(prop.name),
+ .value = utils::toStringView(prop.value),
+ .description = utils::toStringView(prop.description),
+ .supports_expression_language = prop.supports_expression_language
+ });
+ }
+ std::vector<MinifiRelationshipDefinition> relationships;
+ for (auto& rel : Class::Relationships) {
+ relationships.push_back(MinifiRelationshipDefinition{
+ .name = utils::toStringView(rel.name),
+ .description = utils::toStringView(rel.description)
+ });
+ }
+ std::vector<std::vector<MinifiStringView>> attribute_relationships_cache;
+ std::vector<MinifiOutputAttributeDefinition> output_attributes;
+ for (auto& attr : Class::OutputAttributes) {
+ std::vector<MinifiStringView> rel_cache;
+ for (auto& rel : attr.relationships) {
+ rel_cache.push_back(utils::toStringView(rel.name));
+ }
+ output_attributes.push_back(MinifiOutputAttributeDefinition {
+ .name = utils::toStringView(attr.name),
+ .relationships_count = gsl::narrow<uint32_t>(attr.relationships.size()),
+ .relationships_ptr = rel_cache.data(),
+ .description = utils::toStringView(attr.description)
+ });
+ attribute_relationships_cache.push_back(std::move(rel_cache));
+ }
+
+ MinifiProcessorClassDefinition description{
+ .full_name = utils::toStringView(full_name),
+ .description = utils::toStringView(Class::Description),
+ .class_properties_count = gsl::narrow<uint32_t>(class_properties.size()),
+ .class_properties_ptr = class_properties.data(),
+ .dynamic_properties_count = gsl::narrow<uint32_t>(dynamic_properties.size()),
+ .dynamic_properties_ptr = dynamic_properties.data(),
+ .class_relationships_count = gsl::narrow<uint32_t>(relationships.size()),
+ .class_relationships_ptr = relationships.data(),
+ .output_attributes_count = gsl::narrow<uint32_t>(output_attributes.size()),
+ .output_attributes_ptr = output_attributes.data(),
+ .supports_dynamic_properties = Class::SupportsDynamicProperties,
+ .supports_dynamic_relationships = Class::SupportsDynamicRelationships,
+ .input_requirement = utils::toInputRequirement(Class::InputRequirement),
+ .is_single_threaded = Class::IsSingleThreaded,
+
+ .callbacks = MinifiProcessorCallbacks{
+ .create = [] (MinifiProcessorMetadata metadata) -> MINIFI_OWNED void* {
+ return new Class{minifi::core::ProcessorMetadata{
+ .uuid = minifi::utils::Identifier::parse(std::string{metadata.uuid.data, metadata.uuid.length}).value(),
+ .name = std::string{metadata.name.data, metadata.name.length},
+ .logger = std::make_shared<logging::Logger>(metadata.logger)
+ }};
+ },
+ .destroy = [] (MINIFI_OWNED void* self) -> void {
+ delete static_cast<Class*>(self);
+ },
+ .isWorkAvailable = [] (void* self) -> MinifiBool {
+ return static_cast<Class*>(self)->isWorkAvailable();
+ },
+ .restore = [] (void* self, MINIFI_OWNED MinifiFlowFile* ff) -> void {
+ static_cast<Class*>(self)->restore(std::make_shared<FlowFile>(ff));
+ },
+ .getTriggerWhenEmpty = [] (void* self) -> MinifiBool {
+ return static_cast<Class*>(self)->getTriggerWhenEmpty();
+ },
+ .onTrigger = [] (void* self, MinifiProcessContext* context, MinifiProcessSession* session) -> MinifiStatus {
+ ProcessContext context_wrapper(context);
+ ProcessSession session_wrapper(session);
+ try {
+ return static_cast<Class*>(self)->onTrigger(context_wrapper, session_wrapper);
+ } catch (...) {
+ return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+ },
+ .onSchedule = [] (void* self, MinifiProcessContext* context) -> MinifiStatus {
+ ProcessContext context_wrapper(context);
+ try {
+ return static_cast<Class*>(self)->onSchedule(context_wrapper);
+ } catch (...) {
+ return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+ },
+ .onUnSchedule = [] (void* self) -> void {
+ static_cast<Class*>(self)->onUnSchedule();
+ },
+ .calculateMetrics = [] (void* self) -> MINIFI_OWNED MinifiPublishedMetrics* {
+ auto metrics = static_cast<Class*>(self)->calculateMetrics();
+ std::vector<MinifiStringView> names;
+ std::vector<double> values;
+ for (auto& [name, val] : metrics) {
+ names.push_back(utils::toStringView(name));
+ values.push_back(val);
+ }
+ return MinifiPublishedMetricsCreate(gsl::narrow<uint32_t>(metrics.size()), names.data(), values.data());
+ }
+ }
+ };
+
+ fn(description);
+}
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h
similarity index 59%
copy from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
copy to extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h
index 909ba731d..ada9e5178 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
+++ b/extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h
@@ -1,4 +1,5 @@
/**
+ *
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -16,17 +17,26 @@
*/
#pragma once
-#include <memory>
+#include <iostream>
#include <string>
-#include "minifi-cpp/utils/Id.h"
+
+#include "fmt/chrono.h"
+#include "minifi-c.h"
#include "minifi-cpp/core/logging/Logger.h"
-namespace org::apache::nifi::minifi::core {
+namespace org::apache::nifi::minifi::api::core::logging {
+
+class Logger : public minifi::core::logging::Logger {
+ public:
+ explicit Logger(MinifiLogger* impl): impl_(impl) {}
+
+ void set_max_log_size(int size) override;
+ void log_string(minifi::core::logging::LOG_LEVEL level, std::string str) override;
+ bool should_log(minifi::core::logging::LOG_LEVEL level) override;
+ [[nodiscard]] minifi::core::logging::LOG_LEVEL level() const override;
-struct ProcessorMetadata {
- utils::Identifier uuid;
- std::string name;
- std::shared_ptr<logging::Logger> logger;
+ private:
+ MinifiLogger* impl_;
};
-} // namespace org::apache::nifi::minifi::core
+} // namespace org::apache::nifi::minifi::api::core::logging
diff --git a/core-framework/include/io/OutputStream.h b/extension-framework/cpp-extension-lib/include/api/utils/Export.h
similarity index 81%
rename from core-framework/include/io/OutputStream.h
rename to extension-framework/cpp-extension-lib/include/api/utils/Export.h
index c7fae4f5e..b92016cac 100644
--- a/core-framework/include/io/OutputStream.h
+++ b/extension-framework/cpp-extension-lib/include/api/utils/Export.h
@@ -1,5 +1,5 @@
/**
- *
+*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -18,10 +18,12 @@
#pragma once
-#include "minifi-cpp/io/OutputStream.h"
-
-namespace org::apache::nifi::minifi::io {
-
-using OutputStreamImpl = OutputStream;
-
-} // namespace org::apache::nifi::minifi::io
+#ifdef WIN32
+ #ifdef EXTENSION_NAME
+ #define EXTENSIONAPI __declspec(dllexport)
+ #else
+ #define EXTENSIONAPI __declspec(dllimport)
+ #endif
+#else
+ #define EXTENSIONAPI
+#endif
diff --git a/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h b/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h
new file mode 100644
index 000000000..14644b9c7
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <optional>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "api/core/ProcessContext.h"
+#include "utils/Enum.h"
+#include "utils/expected.h"
+#include "utils/OptionalUtils.h"
+#include "utils/ParsingUtils.h"
+#include "minifi-cpp/Exception.h"
+
+namespace org::apache::nifi::minifi::api::utils {
+
+inline std::string parseProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::orThrow(fmt::format("Expected valid value from \"{}::{}\"", ctx.getProcessorName(), property.name));
+}
+
+inline bool parseBoolProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::andThen(parsing::parseBool)
+ | minifi::utils::orThrow(fmt::format("Expected parsable bool from \"{}::{}\"", ctx.getProcessorName(), property.name));
+}
+
+inline uint64_t parseU64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::andThen(parsing::parseIntegral<uint64_t>)
+ | minifi::utils::orThrow(fmt::format("Expected parsable uint64_t from \"{}::{}\"", ctx.getProcessorName(), property.name));
+}
+
+inline int64_t parseI64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::andThen(parsing::parseIntegral<int64_t>)
+ | minifi::utils::orThrow(fmt::format("Expected parsable int64_t from \"{}::{}\"", ctx.getProcessorName(), property.name));
+}
+
+inline std::chrono::milliseconds parseDurationProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::andThen(parsing::parseDuration<std::chrono::milliseconds>)
+ | minifi::utils::orThrow(fmt::format("Expected parsable duration from \"{}::{}\"", ctx.getProcessorName(), property.name));
+}
+
+inline uint64_t parseDataSizeProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::andThen(parsing::parseDataSize)
+ | minifi::utils::orThrow(fmt::format("Expected parsable data size from \"{}::{}\"", ctx.getProcessorName(), property.name));
+}
+
+inline std::optional<std::string> parseOptionalProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ return ctx.getProperty(property.name, flow_file)
+ | minifi::utils::toOptional();
+}
+
+inline std::optional<bool> parseOptionalBoolProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ if (const auto property_str = ctx.getProperty(property.name, flow_file)) {
+ return parsing::parseBool(*property_str)
+ | minifi::utils::orThrow(fmt::format("Expected parsable bool from \"{}::{}\"", ctx.getProcessorName(), property.name));
+ }
+ return std::nullopt;
+}
+
+inline std::optional<uint64_t> parseOptionalU64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ if (const auto property_str = ctx.getProperty(property.name, flow_file)) {
+ if (property_str->empty()) {
+ return std::nullopt;
+ }
+ return parsing::parseIntegral<uint64_t>(*property_str)
+ | minifi::utils::orThrow(fmt::format("Expected parsable uint64_t from \"{}::{}\"", ctx.getProcessorName(), property.name));
+ }
+
+ return std::nullopt;
+}
+
+inline std::optional<int64_t> parseOptionalI64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ if (const auto property_str = ctx.getProperty(property.name, flow_file)) {
+ if (property_str->empty()) {
+ return std::nullopt;
+ }
+ return parsing::parseIntegral<int64_t>(*property_str)
+ | minifi::utils::orThrow(fmt::format("Expected parsable int64_t from \"{}::{}\"", ctx.getProcessorName(), property.name));
+ }
+
+ return std::nullopt;
+}
+
+inline std::optional<std::chrono::milliseconds> parseOptionalDurationProperty(const core::ProcessContext& ctx,
+ const minifi::core::PropertyReference& property,
+ const core::FlowFile* flow_file = nullptr) {
+ if (const auto property_str = ctx.getProperty(property.name, flow_file)) {
+ if (property_str->empty()) {
+ return std::nullopt;
+ }
+ return parsing::parseDuration(*property_str)
+ | minifi::utils::orThrow(fmt::format("Expected parsable duration from \"{}::{}\"", ctx.getProcessorName(), property.name));
+ }
+
+ return std::nullopt;
+}
+
+inline std::optional<uint64_t> parseOptionalDataSizeProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ if (const auto property_str = ctx.getProperty(property.name, flow_file)) {
+ if (property_str->empty()) {
+ return std::nullopt;
+ }
+ return parsing::parseDataSize(*property_str)
+ | minifi::utils::orThrow(fmt::format("Expected parsable data size from \"{}::{}\"", ctx.getProcessorName(), property.name));
+ }
+
+ return std::nullopt;
+}
+
+inline std::optional<float> parseOptionalFloatProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) {
+ if (const auto property_str = ctx.getProperty(property.name, flow_file)) {
+ if (property_str->empty()) {
+ return std::nullopt;
+ }
+ return parsing::parseFloat(*property_str)
+ | minifi::utils::orThrow(fmt::format("Expected parsable float from {}::{}", ctx.getProcessorName(), property.name));
+ }
+ return std::nullopt;
+}
+
+template<typename T>
+T parseEnumProperty(const core::ProcessContext& context, const minifi::core::PropertyReference& prop, const core::FlowFile* flow_file = nullptr) {
+ const auto enum_str = context.getProperty(prop.name, flow_file);
+ if (!enum_str) {
+ throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' is missing");
+ }
+ auto result = magic_enum::enum_cast<T>(*enum_str);
+ if (!result) {
+ throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + *enum_str + "'");
+ }
+ return result.value();
+}
+
+template<typename T>
+std::optional<T> parseOptionalEnumProperty(const core::ProcessContext& context, const minifi::core::PropertyReference& prop) {
+ const auto enum_str = context.getProperty(prop.name);
+
+ if (!enum_str) {
+ return std::nullopt;
+ }
+ auto result = magic_enum::enum_cast<T>(*enum_str);
+ if (!result) {
+ throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + *enum_str + "'");
+ }
+ return result.value();
+}
+
+} // namespace org::apache::nifi::minifi::api::utils
diff --git a/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h b/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h
new file mode 100644
index 000000000..756c52113
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "minifi-c.h"
+#include <string_view>
+#include "minifi-cpp/core/Annotation.h"
+#include "minifi-cpp/utils/gsl.h"
+#include "core/ClassName.h"
+#include "utils/StringUtils.h"
+#include "minifi-cpp/core/PropertyDefinition.h"
+
+namespace org::apache::nifi::minifi::api::utils {
+
+inline MinifiStringView toStringView(std::string_view str) {
+ return MinifiStringView{.data = str.data(), .length = str.length()};
+}
+
+template<typename T>
+std::string classNameWithDots() {
+ std::string class_name{minifi::core::className<T>()};
+ return minifi::utils::string::replaceAll(class_name, "::", ".");
+}
+
+MinifiValidator toStandardPropertyValidator(const minifi::core::PropertyValidator* validator);
+
+inline MinifiInputRequirement toInputRequirement(minifi::core::annotation::Input req) {
+ switch (req) {
+ case minifi::core::annotation::Input::INPUT_REQUIRED: return MINIFI_INPUT_REQUIRED;
+ case minifi::core::annotation::Input::INPUT_ALLOWED: return MINIFI_INPUT_ALLOWED;
+ case minifi::core::annotation::Input::INPUT_FORBIDDEN: return MINIFI_INPUT_FORBIDDEN;
+ }
+ gsl_FailFast();
+}
+
+inline std::vector<MinifiPropertyDefinition> toProperties(std::span<const minifi::core::PropertyReference> props, std::vector<std::vector<MinifiStringView>>& cache) {
+ std::vector<MinifiPropertyDefinition> properties;
+ for (auto& prop : props) {
+ std::vector<MinifiStringView> sv_cache;
+ const size_t allowed_values_begin = sv_cache.size();
+ for (auto& allowed_value : prop.allowed_values) {
+ sv_cache.emplace_back(toStringView(allowed_value));
+ }
+ const std::optional<size_t> allowed_types_begin = [&] () -> std::optional<size_t> {
+ if (prop.allowed_types.empty()) {
+ return std::nullopt;
+ }
+ gsl_Expects(prop.allowed_types.size() == 1);
+ sv_cache.emplace_back(toStringView(prop.allowed_types[0]));
+ return sv_cache.size() - 1;
+ }();
+ const std::optional<size_t> default_value_begin = [&] () -> std::optional<size_t> {
+ if (!prop.default_value) {
+ return std::nullopt;
+ }
+ sv_cache.emplace_back(toStringView(*prop.default_value));
+ return sv_cache.size() - 1;
+ }();
+ properties.push_back(MinifiPropertyDefinition{
+ .name = toStringView(prop.name),
+ .display_name = toStringView(prop.display_name),
+ .description = toStringView(prop.description),
+ .is_required = prop.is_required,
+ .is_sensitive = prop.is_sensitive,
+
+ .default_value = default_value_begin ? sv_cache.data() + default_value_begin.value() : nullptr,
+ .allowed_values_count = prop.allowed_values.size(),
+ .allowed_values_ptr = sv_cache.data() + allowed_values_begin,
+ .validator = toStandardPropertyValidator(prop.validator),
+
+ .type = allowed_types_begin ? sv_cache.data() + allowed_types_begin.value() : nullptr,
+ .supports_expression_language = prop.supports_expression_language
+ });
+ cache.emplace_back(std::move(sv_cache));
+ }
+ return properties;
+}
+
+std::error_code make_error_code(MinifiStatus status);
+
+} // namespace org::apache::nifi::minifi::api::utils
diff --git a/extensions/llamacpp/CMakeLists.txt b/extension-framework/cpp-extension-lib/libtest/CMakeLists.txt
similarity index 56%
copy from extensions/llamacpp/CMakeLists.txt
copy to extension-framework/cpp-extension-lib/libtest/CMakeLists.txt
index e10bea82b..a75111901 100644
--- a/extensions/llamacpp/CMakeLists.txt
+++ b/extension-framework/cpp-extension-lib/libtest/CMakeLists.txt
@@ -1,5 +1,5 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
+#
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
@@ -17,21 +17,6 @@
# under the License.
#
-if (NOT (ENABLE_ALL OR ENABLE_LLAMACPP))
- return()
-endif()
-
-include(LlamaCpp)
-
-include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
-
-file(GLOB SOURCES "processors/*.cpp")
-
-add_minifi_library(minifi-llamacpp SHARED ${SOURCES})
-target_include_directories(minifi-llamacpp PUBLIC "${CMAKE_SOURCE_DIR}/extensions/llamacpp")
-target_include_directories(minifi-llamacpp PUBLIC "${LLAMACPP_INCLUDE_DIRS}")
-
-target_link_libraries(minifi-llamacpp ${LIBMINIFI} llama)
-
-register_extension(minifi-llamacpp "LLAMACPP EXTENSION" LLAMACPP-EXTENSION "Provides llama.cpp support" "extensions/llamacpp/tests")
-
+add_library(libminifi-c-unittest INTERFACE)
+target_link_libraries(libminifi-c-unittest INTERFACE minifi-cpp-extension-lib core-minifi Catch2WithMain)
+target_include_directories(libminifi-c-unittest INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
diff --git a/extension-framework/cpp-extension-lib/libtest/CProcessorTestUtils.h b/extension-framework/cpp-extension-lib/libtest/CProcessorTestUtils.h
new file mode 100644
index 000000000..142f4709e
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/libtest/CProcessorTestUtils.h
@@ -0,0 +1,38 @@
+/**
+*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "core/Processor.h"
+#include "minifi-cpp/core/ProcessorMetadata.h"
+#include "api/core/Resource.h"
+#include "utils/CProcessor.h"
+
+namespace org::apache::nifi::minifi::test::utils {
+
+template<typename T, typename ...Args>
+std::unique_ptr<minifi::core::Processor> make_custom_c_processor(minifi::core::ProcessorMetadata metadata, Args&&... args) { // NOLINT(cppcoreguidelines-missing-std-forward)
+ std::unique_ptr<minifi::core::ProcessorApi> processor_impl;
+ minifi::api::core::useProcessorClassDescription<T>([&] (const MinifiProcessorClassDefinition& description) {
+ minifi::utils::useCProcessorClassDescription(description, [&] (const auto&, auto c_description) {
+ processor_impl = std::make_unique<minifi::utils::CProcessor>(std::move(c_description), metadata, new T(metadata, std::forward<Args>(args)...));
+ });
+ });
+ return std::make_unique<minifi::core::Processor>(metadata.name, metadata.uuid, std::move(processor_impl));
+}
+
+} // namespace org::apache::nifi::minifi::test::utils
diff --git a/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp b/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp
new file mode 100644
index 000000000..76ed2231b
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "api/core/ProcessContext.h"
+#include "api/utils/minifi-c-utils.h"
+#include "api/core/FlowFile.h"
+
+namespace org::apache::nifi::minifi::api::core {
+
+nonstd::expected<std::string, std::error_code> ProcessContext::getProperty(std::string_view name, const FlowFile* flow_file) const {
+ std::optional<std::string> value;
+ MinifiStatus status = MinifiProcessContextGetProperty(impl_, utils::toStringView(name), flow_file ? flow_file->get() : MINIFI_NULL,
+ [] (void* data, MinifiStringView result) {
+ (*static_cast<std::optional<std::string>*>(data)) = std::string(result.data, result.length);
+ }, &value);
+
+ if (!value) {
+ return nonstd::make_unexpected(utils::make_error_code(status));
+ }
+ return value.value();
+}
+
+bool ProcessContext::hasNonEmptyProperty(std::string_view name) const {
+ return MinifiProcessContextHasNonEmptyProperty(impl_, utils::toStringView(name));
+}
+
+std::string ProcessContext::getProcessorName() const {
+ std::string result;
+ MinifiProcessContextGetProcessorName(impl_, [] (void* data, MinifiStringView name) {
+ *static_cast<std::string*>(data) = std::string(name.data, name.length);
+ }, &result);
+ return result;
+}
+
+
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp b/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp
new file mode 100644
index 000000000..c5c51765c
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp
@@ -0,0 +1,153 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "api/core/ProcessSession.h"
+
+#include "io/InputStream.h"
+#include "io/OutputStream.h"
+#include "api/utils/minifi-c-utils.h"
+#include "api/core/FlowFile.h"
+#include "minifi-cpp/Exception.h"
+
+namespace org::apache::nifi::minifi::api::core {
+
+namespace {
+
+class MinifiOutputStreamWrapper : public io::OutputStreamImpl {
+ public:
+ explicit MinifiOutputStreamWrapper(MinifiOutputStream* impl): impl_(impl) {}
+
+ size_t write(const uint8_t *value, size_t len) override {
+ return MinifiOutputStreamWrite(impl_, reinterpret_cast<const char*>(value), len);
+ }
+
+ void close() override {gsl_FailFast();}
+ void seek(size_t /*offset*/) override {gsl_FailFast();}
+ [[nodiscard]] size_t tell() const override {gsl_FailFast();}
+ int initialize() override {gsl_FailFast();}
+ [[nodiscard]] std::span<const std::byte> getBuffer() const override {gsl_FailFast();}
+
+ private:
+ MinifiOutputStream* impl_;
+};
+
+class MinifiInputStreamWrapper : public io::InputStreamImpl {
+ public:
+ explicit MinifiInputStreamWrapper(MinifiInputStream* impl): impl_(impl) {}
+
+ size_t read(std::span<std::byte> out_buffer) override {
+ return MinifiInputStreamRead(impl_, reinterpret_cast<char*>(out_buffer.data()), out_buffer.size());
+ }
+
+ [[nodiscard]] size_t size() const override {
+ return MinifiInputStreamSize(impl_);
+ }
+
+ void close() override {gsl_FailFast();}
+ void seek(size_t /*offset*/) override {gsl_FailFast();}
+ [[nodiscard]] size_t tell() const override {gsl_FailFast();}
+ int initialize() override {gsl_FailFast();}
+ [[nodiscard]] std::span<const std::byte> getBuffer() const override {gsl_FailFast();}
+
+ private:
+ MinifiInputStream* impl_;
+};
+
+} // namespace
+
+FlowFile ProcessSession::get() {
+ return FlowFile{MinifiProcessSessionGet(impl_)};
+}
+
+FlowFile ProcessSession::create(const FlowFile* parent) {
+ return FlowFile{MinifiProcessSessionCreate(impl_, parent ? parent->get() : MINIFI_NULL)};
+}
+
+void ProcessSession::transfer(FlowFile ff, const minifi::core::Relationship& relationship) {
+ const auto rel_name = relationship.getName();
+ MinifiProcessSessionTransfer(impl_, ff.release(), utils::toStringView(rel_name));
+}
+
+void ProcessSession::remove(FlowFile ff) {
+ MinifiProcessSessionRemove(impl_, ff.release());
+}
+
+void ProcessSession::write(FlowFile& flow_file, const io::OutputStreamCallback& callback) {
+ const auto status = MinifiProcessSessionWrite(impl_, flow_file.get(), [] (void* data, MinifiOutputStream* output) {
+ return (*static_cast<const io::OutputStreamCallback*>(data))(std::make_shared<MinifiOutputStreamWrapper>(output));
+ }, const_cast<io::OutputStreamCallback*>(&callback));
+ if (status != MINIFI_STATUS_SUCCESS) {
+ throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to process flowfile content");
+ }
+}
+
+void ProcessSession::read(FlowFile& flow_file, const io::InputStreamCallback& callback) {
+ const auto status = MinifiProcessSessionRead(impl_, flow_file.get(), [] (void* data, MinifiInputStream* input) {
+ return (*static_cast<const io::InputStreamCallback*>(data))(std::make_shared<MinifiInputStreamWrapper>(input));
+ }, const_cast<io::InputStreamCallback*>(&callback));
+ if (status != MINIFI_STATUS_SUCCESS) {
+ throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to process flowfile content");
+ }
+}
+
+void ProcessSession::setAttribute(FlowFile& ff, std::string_view key, std::string value) { // NOLINT(performance-unnecessary-value-param)
+ MinifiStringView value_ref = utils::toStringView(value);
+ MinifiFlowFileSetAttribute(impl_, ff.get(), utils::toStringView(key), &value_ref);
+}
+
+void ProcessSession::removeAttribute(FlowFile& ff, std::string_view key) {
+ MinifiFlowFileSetAttribute(impl_, ff.get(), utils::toStringView(key), nullptr);
+}
+
+std::optional<std::string> ProcessSession::getAttribute(FlowFile& ff, std::string_view key) {
+ std::optional<std::string> result;
+ MinifiFlowFileGetAttribute(impl_, ff.get(), utils::toStringView(key), [] (void* user_ctx, MinifiStringView value) {
+ *static_cast<std::optional<std::string>*>(user_ctx) = std::string{value.data, value.length};
+ }, &result);
+ return result;
+}
+
+std::map<std::string, std::string> ProcessSession::getAttributes(FlowFile& ff) {
+ std::map<std::string, std::string> result;
+ MinifiFlowFileGetAttributes(impl_, ff.get(), [] (void* user_ctx, MinifiStringView value, MinifiStringView key) {
+ static_cast<std::map<std::string, std::string>*>(user_ctx)->insert({std::string{value.data, value.length}, std::string{key.data, key.length}});
+ }, &result);
+ return result;
+}
+
+void ProcessSession::writeBuffer(FlowFile& flow_file, std::span<const char> buffer) {
+ writeBuffer(flow_file, as_bytes(buffer));
+}
+
+void ProcessSession::writeBuffer(FlowFile& flow_file, std::span<const std::byte> buffer) {
+ write(flow_file, [buffer](const std::shared_ptr<io::OutputStream>& output_stream) {
+ const auto write_status = output_stream->write(buffer);
+ return io::isError(write_status) ? -1 : gsl::narrow<int64_t>(write_status);
+ });
+}
+
+std::vector<std::byte> ProcessSession::readBuffer(FlowFile& flow_file) {
+ std::vector<std::byte> result;
+ read(flow_file, [&result](const std::shared_ptr<io::InputStream>& input_stream) {
+ result.resize(input_stream->size());
+ const auto read_status = input_stream->read(result);
+ return io::isError(read_status) ? -1 : gsl::narrow<int64_t>(read_status);
+ });
+ return result;
+}
+
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/core-framework/src/core/ProcessorImpl.cpp b/extension-framework/cpp-extension-lib/src/core/ProcessorImpl.cpp
similarity index 54%
copy from core-framework/src/core/ProcessorImpl.cpp
copy to extension-framework/cpp-extension-lib/src/core/ProcessorImpl.cpp
index c45c0ff8f..4689bc942 100644
--- a/core-framework/src/core/ProcessorImpl.cpp
+++ b/extension-framework/cpp-extension-lib/src/core/ProcessorImpl.cpp
@@ -1,7 +1,4 @@
/**
- * @file Processor.cpp
- * Processor class implementation
- *
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,9 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#include "core/ProcessorImpl.h"
+#include "api/core/ProcessorImpl.h"
-#include <ctime>
#include <cctype>
#include <memory>
@@ -27,22 +23,13 @@
#include <string>
#include <vector>
-#include "minifi-cpp/Connection.h"
-#include "core/logging/LoggerFactory.h"
-#include "minifi-cpp/core/ProcessorConfig.h"
-#include "minifi-cpp/core/ProcessContext.h"
-#include "minifi-cpp/core/ProcessSessionFactory.h"
-#include "minifi-cpp/utils/gsl.h"
-#include "range/v3/algorithm/any_of.hpp"
#include "fmt/format.h"
-#include "minifi-cpp/Exception.h"
-#include "minifi-cpp/core/ProcessorDescriptor.h"
using namespace std::literals::chrono_literals;
-namespace org::apache::nifi::minifi::core {
+namespace org::apache::nifi::minifi::api::core {
-ProcessorImpl::ProcessorImpl(ProcessorMetadata metadata)
+ProcessorImpl::ProcessorImpl(minifi::core::ProcessorMetadata metadata)
: metadata_(std::move(metadata)),
trigger_when_empty_(false),
logger_(metadata_.logger) {
@@ -58,7 +45,7 @@ bool ProcessorImpl::isWorkAvailable() {
}
void ProcessorImpl::restore(const std::shared_ptr<FlowFile>& /*file*/) {
- gsl_Assert("Not implemented");
+ gsl_Assert(false && "Not implemented");
}
std::string ProcessorImpl::getName() const {
@@ -73,25 +60,23 @@ utils::SmallString<36> ProcessorImpl::getUUIDStr() const {
return getUUID().to_string();
}
-void ProcessorImpl::initialize(ProcessorDescriptor& self) {
- gsl_Expects(!descriptor_);
- descriptor_ = &self;
- auto guard = gsl::finally([&] {descriptor_ = nullptr;});
- initialize();
-}
-
-void ProcessorImpl::setSupportedRelationships(std::span<const RelationshipDefinition> relationships) {
- gsl_Expects(descriptor_);
- descriptor_->setSupportedRelationships(relationships);
+MinifiStatus ProcessorImpl::onSchedule(ProcessContext& ctx) {
+ try {
+ return onScheduleImpl(ctx);
+ } catch (const std::exception& e) {
+ logger_->log_error("{}", e.what());
+ throw;
+ }
}
-void ProcessorImpl::setSupportedProperties(std::span<const PropertyReference> properties) {
- gsl_Expects(descriptor_);
- descriptor_->setSupportedProperties(properties);
+MinifiStatus ProcessorImpl::onTrigger(ProcessContext& ctx, ProcessSession& session) {
+ try {
+ return onTriggerImpl(ctx, session);
+ } catch (const std::exception& e) {
+ logger_->log_error("{}", e.what());
+ throw;
+ }
}
-void ProcessorImpl::setLoggerCallback(const std::function<void(logging::LOG_LEVEL level, const std::string& message)>& callback) {
- logger_->setLogCallback(callback);
-}
-} // namespace org::apache::nifi::minifi::core
+} // namespace org::apache::nifi::minifi::api::core
diff --git a/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp b/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp
new file mode 100644
index 000000000..d91a67f60
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp
@@ -0,0 +1,69 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "api/core/logging/Logger.h"
+
+namespace org::apache::nifi::minifi::api::core::logging {
+
+namespace {
+
+MinifiLogLevel toCLogLevel(minifi::core::logging::LOG_LEVEL lvl) {
+ switch (lvl) {
+ case minifi::core::logging::trace: return MINIFI_LOG_LEVEL_TRACE;
+ case minifi::core::logging::debug: return MINIFI_LOG_LEVEL_DEBUG;
+ case minifi::core::logging::info: return MINIFI_LOG_LEVEL_INFO;
+ case minifi::core::logging::warn: return MINIFI_LOG_LEVEL_WARNING;
+ case minifi::core::logging::err: return MINIFI_LOG_LEVEL_ERROR;
+ case minifi::core::logging::critical: return MINIFI_LOG_LEVEL_CRITICAL;
+ case minifi::core::logging::off: return MINIFI_LOG_LEVEL_OFF;
+ }
+ gsl_FailFast();
+}
+
+minifi::core::logging::LOG_LEVEL toLogLevel(MinifiLogLevel level) {
+ switch (level) {
+ case MINIFI_LOG_LEVEL_TRACE: return minifi::core::logging::trace;
+ case MINIFI_LOG_LEVEL_DEBUG: return minifi::core::logging::debug;
+ case MINIFI_LOG_LEVEL_INFO: return minifi::core::logging::info;
+ case MINIFI_LOG_LEVEL_WARNING: return minifi::core::logging::warn;
+ case MINIFI_LOG_LEVEL_ERROR: return minifi::core::logging::err;
+ case MINIFI_LOG_LEVEL_CRITICAL: return minifi::core::logging::critical;
+ case MINIFI_LOG_LEVEL_OFF: return minifi::core::logging::off;
+ }
+ gsl_FailFast();
+}
+
+} // namespace
+
+void Logger::set_max_log_size(int size) {
+ MinifiLoggerSetMaxLogSize(impl_, size);
+}
+
+void Logger::log_string(minifi::core::logging::LOG_LEVEL level, std::string str) {
+ MinifiLoggerLogString(impl_, toCLogLevel(level), MinifiStringView{.data = str.data(), .length = str.length()});
+}
+
+bool Logger::should_log(minifi::core::logging::LOG_LEVEL level) {
+ return MinifiLoggerShouldLog(impl_, toCLogLevel(level));
+}
+
+[[nodiscard]] minifi::core::logging::LOG_LEVEL Logger::level() const {
+ return toLogLevel(MinifiLoggerLevel(impl_));
+}
+
+
+} // namespace org::apache::nifi::minifi::api::core::logging
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/extension-framework/cpp-extension-lib/src/utils/ParsingErrors.cpp
similarity index 68%
copy from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
copy to extension-framework/cpp-extension-lib/src/utils/ParsingErrors.cpp
index 909ba731d..3848b0f53 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
+++ b/extension-framework/cpp-extension-lib/src/utils/ParsingErrors.cpp
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -14,19 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#pragma once
-#include <memory>
-#include <string>
-#include "minifi-cpp/utils/Id.h"
-#include "minifi-cpp/core/logging/Logger.h"
+#include "utils/ParsingErrors.h"
namespace org::apache::nifi::minifi::core {
-struct ProcessorMetadata {
- utils::Identifier uuid;
- std::string name;
- std::shared_ptr<logging::Logger> logger;
+const minifi::core::ParsingErrorCategory& parsing_error_category() noexcept {
+ static minifi::core::ParsingErrorCategory category;
+ return category;
};
+std::error_code make_error_code(minifi::core::ParsingErrorCode c) {
+ return {static_cast<int>(c), parsing_error_category()};
+}
+
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/extension-framework/cpp-extension-lib/src/utils/PropertyErrors.cpp
similarity index 68%
copy from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
copy to extension-framework/cpp-extension-lib/src/utils/PropertyErrors.cpp
index 909ba731d..f38378cfe 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
+++ b/extension-framework/cpp-extension-lib/src/utils/PropertyErrors.cpp
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -14,19 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#pragma once
-#include <memory>
-#include <string>
-#include "minifi-cpp/utils/Id.h"
-#include "minifi-cpp/core/logging/Logger.h"
+#include "utils/PropertyErrors.h"
namespace org::apache::nifi::minifi::core {
-struct ProcessorMetadata {
- utils::Identifier uuid;
- std::string name;
- std::shared_ptr<logging::Logger> logger;
+const minifi::core::PropertyErrorCategory& property_error_category() noexcept {
+ static minifi::core::PropertyErrorCategory category;
+ return category;
};
+std::error_code make_error_code(minifi::core::PropertyErrorCode c) {
+ return {static_cast<int>(c), property_error_category()};
+}
+
} // namespace org::apache::nifi::minifi::core
diff --git a/extension-framework/cpp-extension-lib/src/utils/minifi-c-utils.cpp b/extension-framework/cpp-extension-lib/src/utils/minifi-c-utils.cpp
new file mode 100644
index 000000000..a2690d170
--- /dev/null
+++ b/extension-framework/cpp-extension-lib/src/utils/minifi-c-utils.cpp
@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "api/utils/minifi-c-utils.h"
+#include "utils/PropertyErrors.h"
+
+namespace org::apache::nifi::minifi::api::utils {
+
+std::error_code make_error_code(MinifiStatus status) {
+ switch (status) {
+ case MINIFI_STATUS_NOT_SUPPORTED_PROPERTY: return make_error_code(minifi::core::PropertyErrorCode::NotSupportedProperty);
+ case MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED: return make_error_code(minifi::core::PropertyErrorCode::DynamicPropertiesNotSupported);
+ case MINIFI_STATUS_PROPERTY_NOT_SET: return make_error_code(minifi::core::PropertyErrorCode::PropertyNotSet);
+ case MINIFI_STATUS_VALIDATION_FAILED: return make_error_code(minifi::core::PropertyErrorCode::ValidationFailed);
+ default: return std::error_code{};
+ }
+}
+
+MinifiValidator toStandardPropertyValidator(const minifi::core::PropertyValidator* validator) {
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::ALWAYS_VALID_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_ALWAYS_VALID;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::NON_BLANK_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_NON_BLANK;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::TIME_PERIOD_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_TIME_PERIOD;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::BOOLEAN_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_BOOLEAN;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::INTEGER_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_INTEGER;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::UNSIGNED_INTEGER_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_UNSIGNED_INTEGER;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::DATA_SIZE_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_DATA_SIZE;
+ }
+ if (validator->getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::PORT_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ return MINIFI_VALIDATOR_PORT;
+ }
+ gsl_FailFast();
+}
+
+} // namespace org::apache::nifi::minifi::api::utils
diff --git a/extension-framework/include/core/ProcessSession.h b/extension-framework/include/core/ProcessSession.h
index 26678bedb..672144766 100644
--- a/extension-framework/include/core/ProcessSession.h
+++ b/extension-framework/include/core/ProcessSession.h
@@ -24,4 +24,3 @@ namespace org::apache::nifi::minifi::core::detail {
std::string to_string(const ReadBufferResult& read_buffer_result);
} // namespace org::apache::nifi::minifi::core::detail
-
diff --git a/core-framework/include/utils/minifi-c-utils.h b/extension-framework/include/utils/ExtensionInitUtils.h
similarity index 97%
rename from core-framework/include/utils/minifi-c-utils.h
rename to extension-framework/include/utils/ExtensionInitUtils.h
index a23a3da65..1af7617e5 100644
--- a/core-framework/include/utils/minifi-c-utils.h
+++ b/extension-framework/include/utils/ExtensionInitUtils.h
@@ -16,11 +16,11 @@
*/
#pragma once
-#include "minifi-c/minifi-c.h"
+#include <string>
#include <string_view>
#include <optional>
#include <functional>
-#include "minifi-cpp/utils/gsl.h"
+#include "minifi-c/minifi-c.h"
namespace org::apache::nifi::minifi::utils {
diff --git a/extension-framework/src/utils/file/FileWriterCallback.cpp b/extension-framework/src/utils/file/FileWriterCallback.cpp
index 3a1542efc..7abfe8792 100644
--- a/extension-framework/src/utils/file/FileWriterCallback.cpp
+++ b/extension-framework/src/utils/file/FileWriterCallback.cpp
@@ -16,8 +16,11 @@
*/
#include "utils/file/FileWriterCallback.h"
+
#include <fstream>
+#include "utils/Id.h"
+
namespace org::apache::nifi::minifi::utils {
FileWriterCallback::FileWriterCallback(std::filesystem::path dest_path)
diff --git a/extensions/llamacpp/CMakeLists.txt b/extensions/llamacpp/CMakeLists.txt
index e10bea82b..421143f69 100644
--- a/extensions/llamacpp/CMakeLists.txt
+++ b/extensions/llamacpp/CMakeLists.txt
@@ -31,7 +31,7 @@ add_minifi_library(minifi-llamacpp SHARED ${SOURCES})
target_include_directories(minifi-llamacpp PUBLIC "${CMAKE_SOURCE_DIR}/extensions/llamacpp")
target_include_directories(minifi-llamacpp PUBLIC "${LLAMACPP_INCLUDE_DIRS}")
-target_link_libraries(minifi-llamacpp ${LIBMINIFI} llama)
+target_link_libraries(minifi-llamacpp minifi-cpp-extension-lib llama)
-register_extension(minifi-llamacpp "LLAMACPP EXTENSION" LLAMACPP-EXTENSION "Provides llama.cpp support" "extensions/llamacpp/tests")
+register_c_api_extension(minifi-llamacpp "LLAMACPP EXTENSION" LLAMACPP-EXTENSION "Provides llama.cpp support" "extensions/llamacpp/tests")
diff --git a/extensions/llamacpp/processors/DefaultLlamaContext.cpp b/extensions/llamacpp/processors/DefaultLlamaContext.cpp
index b453110b8..cc8fb017d 100644
--- a/extensions/llamacpp/processors/DefaultLlamaContext.cpp
+++ b/extensions/llamacpp/processors/DefaultLlamaContext.cpp
@@ -18,7 +18,6 @@
#include "DefaultLlamaContext.h"
#include "minifi-cpp/Exception.h"
#include "fmt/format.h"
-#include "utils/ConfigurationUtils.h"
namespace org::apache::nifi::minifi::extensions::llamacpp::processors {
@@ -36,6 +35,9 @@ std::vector<llama_token> tokenizeInput(const llama_vocab* vocab, const std::stri
}
return tokenized_input;
}
+
+constexpr size_t DEFAULT_BUFFER_SIZE = 4096;
+
} // namespace
@@ -88,7 +90,7 @@ std::optional<std::string> DefaultLlamaContext::applyTemplate(const std::vector<
std::transform(messages.begin(), messages.end(), std::back_inserter(llama_messages),
[](const LlamaChatMessage& msg) { return llama_chat_message{.role = msg.role.c_str(), .content = msg.content.c_str()}; });
std::string text;
- text.resize(utils::configuration::DEFAULT_BUFFER_SIZE);
+ text.resize(DEFAULT_BUFFER_SIZE);
const char * chat_template = llama_model_chat_template(llama_model_, nullptr);
int32_t res_size = llama_chat_apply_template(chat_template, llama_messages.data(), llama_messages.size(), true, text.data(), gsl::narrow<int32_t>(text.size()));
if (res_size < 0) {
diff --git a/extensions/llamacpp/processors/ExtensionInitializer.cpp b/extensions/llamacpp/processors/ExtensionInitializer.cpp
new file mode 100644
index 000000000..40508955c
--- /dev/null
+++ b/extensions/llamacpp/processors/ExtensionInitializer.cpp
@@ -0,0 +1,43 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "RunLlamaCppInference.h"
+#include "api/core/Resource.h"
+#include "api/utils/minifi-c-utils.h"
+
+#define MKSOC(x) #x
+#define MAKESTRING(x) MKSOC(x) // NOLINT(cppcoreguidelines-macro-usage)
+
+namespace minifi = org::apache::nifi::minifi;
+
+extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+ MinifiExtension* extension = nullptr;
+ minifi::api::core::useProcessorClassDescription<minifi::extensions::llamacpp::processors::RunLlamaCppInference>([&] (const MinifiProcessorClassDefinition& description) {
+ MinifiExtensionCreateInfo ext_create_info{
+ .name = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_NAME)),
+ .version = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_VERSION)),
+ .deinit = nullptr,
+ .user_data = nullptr,
+ .processors_count = 1,
+ .processors_ptr = &description,
+ };
+ extension = MinifiCreateExtension(minifi::api::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+ });
+ return extension;
+}
+
+extern const char* const MINIFI_API_VERSION_TAG_var = MINIFI_API_VERSION_TAG;
diff --git a/extensions/llamacpp/processors/RunLlamaCppInference.cpp b/extensions/llamacpp/processors/RunLlamaCppInference.cpp
index 2524cfb8a..5927a0199 100644
--- a/extensions/llamacpp/processors/RunLlamaCppInference.cpp
+++ b/extensions/llamacpp/processors/RunLlamaCppInference.cpp
@@ -16,87 +16,78 @@
*/
#include "RunLlamaCppInference.h"
-#include "minifi-cpp/core/ProcessContext.h"
-#include "core/ProcessSession.h"
-#include "core/Resource.h"
-#include "minifi-cpp/Exception.h"
+#include "api/core/ProcessContext.h"
+#include "api/core/ProcessSession.h"
+#include "api/core/Resource.h"
#include "rapidjson/document.h"
#include "rapidjson/error/en.h"
#include "LlamaContext.h"
-#include "utils/ProcessorConfigUtils.h"
+#include "api/utils/ProcessorConfigUtils.h"
#include "DefaultLlamaContext.h"
namespace org::apache::nifi::minifi::extensions::llamacpp::processors {
-void RunLlamaCppInference::initialize() {
- setSupportedProperties(Properties);
- setSupportedRelationships(Relationships);
-}
-
-void RunLlamaCppInference::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) {
+MinifiStatus RunLlamaCppInference::onScheduleImpl(api::core::ProcessContext& context) {
model_path_.clear();
- model_path_ = utils::parseProperty(context, ModelPath);
+ model_path_ = api::utils::parseProperty(context, ModelPath);
system_prompt_ = context.getProperty(SystemPrompt).value_or("");
LlamaSamplerParams llama_sampler_params;
- llama_sampler_params.temperature = utils::parseOptionalFloatProperty(context, Temperature);
- if (auto top_k = utils::parseOptionalI64Property(context, TopK)) {
+ llama_sampler_params.temperature = api::utils::parseOptionalFloatProperty(context, Temperature);
+ if (auto top_k = api::utils::parseOptionalI64Property(context, TopK)) {
llama_sampler_params.top_k = gsl::narrow<int32_t>(*top_k);
}
- llama_sampler_params.top_p = utils::parseOptionalFloatProperty(context, TopP);
- llama_sampler_params.min_p = utils::parseOptionalFloatProperty(context, MinP);
- llama_sampler_params.min_keep = utils::parseU64Property(context, MinKeep);
+ llama_sampler_params.top_p = api::utils::parseOptionalFloatProperty(context, TopP);
+ llama_sampler_params.min_p = api::utils::parseOptionalFloatProperty(context, MinP);
+ llama_sampler_params.min_keep = api::utils::parseU64Property(context, MinKeep);
LlamaContextParams llama_ctx_params;
- llama_ctx_params.n_ctx = gsl::narrow<uint32_t>(utils::parseU64Property(context, TextContextSize));
- llama_ctx_params.n_batch = gsl::narrow<uint32_t>(utils::parseU64Property(context, LogicalMaximumBatchSize));
- llama_ctx_params.n_ubatch = gsl::narrow<uint32_t>(utils::parseU64Property(context, PhysicalMaximumBatchSize));
- llama_ctx_params.n_seq_max = gsl::narrow<uint32_t>(utils::parseU64Property(context, MaxNumberOfSequences));
- llama_ctx_params.n_threads = gsl::narrow<int32_t>(utils::parseI64Property(context, ThreadsForGeneration));
- llama_ctx_params.n_threads_batch = gsl::narrow<int32_t>(utils::parseI64Property(context, ThreadsForBatchProcessing));
+ llama_ctx_params.n_ctx = gsl::narrow<uint32_t>(api::utils::parseU64Property(context, TextContextSize));
+ llama_ctx_params.n_batch = gsl::narrow<uint32_t>(api::utils::parseU64Property(context, LogicalMaximumBatchSize));
+ llama_ctx_params.n_ubatch = gsl::narrow<uint32_t>(api::utils::parseU64Property(context, PhysicalMaximumBatchSize));
+ llama_ctx_params.n_seq_max = gsl::narrow<uint32_t>(api::utils::parseU64Property(context, MaxNumberOfSequences));
+ llama_ctx_params.n_threads = gsl::narrow<int32_t>(api::utils::parseI64Property(context, ThreadsForGeneration));
+ llama_ctx_params.n_threads_batch = gsl::narrow<int32_t>(api::utils::parseI64Property(context, ThreadsForBatchProcessing));
if (llama_context_provider_) {
llama_ctx_ = llama_context_provider_(model_path_, llama_sampler_params, llama_ctx_params);
} else {
llama_ctx_ = std::make_unique<DefaultLlamaContext>(model_path_, llama_sampler_params, llama_ctx_params);
}
+
+ return MINIFI_STATUS_SUCCESS;
}
void RunLlamaCppInference::increaseTokensIn(uint64_t token_count) {
- auto* const llamacpp_metrics = dynamic_cast<RunLlamaCppInferenceMetrics*>(metrics_extension_.get());
- gsl_Assert(llamacpp_metrics);
- llamacpp_metrics->tokens_in += token_count;
+ metrics_.tokens_in += token_count;
}
void RunLlamaCppInference::increaseTokensOut(uint64_t token_count) {
- auto* const llamacpp_metrics = dynamic_cast<RunLlamaCppInferenceMetrics*>(metrics_extension_.get());
- gsl_Assert(llamacpp_metrics);
- llamacpp_metrics->tokens_out += token_count;
+ metrics_.tokens_out += token_count;
}
-void RunLlamaCppInference::onTrigger(core::ProcessContext& context, core::ProcessSession& session) {
+MinifiStatus RunLlamaCppInference::onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) {
auto flow_file = session.get();
if (!flow_file) {
- context.yield();
- return;
+ return MINIFI_STATUS_PROCESSOR_YIELD;
}
- auto prompt = context.getProperty(Prompt, flow_file.get()).value_or("");
+ auto prompt = context.getProperty(Prompt, &flow_file).value_or("");
auto read_result = session.readBuffer(flow_file);
std::string input_data_and_prompt;
- if (!read_result.buffer.empty()) {
+ if (!read_result.empty()) {
input_data_and_prompt.append("Input data (or flow file content):\n");
- input_data_and_prompt.append({reinterpret_cast<const char*>(read_result.buffer.data()), read_result.buffer.size()});
+ input_data_and_prompt.append({reinterpret_cast<const char*>(read_result.data()), read_result.size()});
input_data_and_prompt.append("\n\n");
}
input_data_and_prompt.append(prompt);
if (input_data_and_prompt.empty()) {
logger_->log_error("Input data and prompt are empty");
- session.transfer(flow_file, Failure);
- return;
+ session.transfer(std::move(flow_file), Failure);
+ return MINIFI_STATUS_SUCCESS;
}
auto input = [&] {
@@ -111,8 +102,8 @@ void RunLlamaCppInference::onTrigger(core::ProcessContext& context, core::Proces
if (!input) {
logger_->log_error("Inference failed with while applying template");
- session.transfer(flow_file, Failure);
- return;
+ session.transfer(std::move(flow_file), Failure);
+ return MINIFI_STATUS_SUCCESS;
}
logger_->log_debug("AI model input: {}", *input);
@@ -128,8 +119,8 @@ void RunLlamaCppInference::onTrigger(core::ProcessContext& context, core::Proces
if (!generation_result) {
logger_->log_error("Inference failed with generation error: '{}'", generation_result.error());
- session.transfer(flow_file, Failure);
- return;
+ session.transfer(std::move(flow_file), Failure);
+ return MINIFI_STATUS_SUCCESS;
}
increaseTokensIn(generation_result->num_tokens_in);
@@ -139,17 +130,17 @@ void RunLlamaCppInference::onTrigger(core::ProcessContext& context, core::Proces
logger_->log_debug("AI model inference time: {} ms", elapsed_time);
logger_->log_debug("AI model output: {}", text);
- flow_file->setAttribute(LlamaCppTimeToFirstToken.name, std::to_string(generation_result->time_to_first_token.count()) + " ms");
- flow_file->setAttribute(LlamaCppTokensPerSecond.name, fmt::format("{:.2f}", generation_result->tokens_per_second));
+ session.setAttribute(flow_file, LlamaCppTimeToFirstToken.name, std::to_string(generation_result->time_to_first_token.count()) + " ms");
+ session.setAttribute(flow_file, LlamaCppTokensPerSecond.name, fmt::format("{:.2f}", generation_result->tokens_per_second));
session.writeBuffer(flow_file, text);
- session.transfer(flow_file, Success);
+ session.transfer(std::move(flow_file), Success);
+
+ return MINIFI_STATUS_SUCCESS;
}
-void RunLlamaCppInference::notifyStop() {
+void RunLlamaCppInference::onUnSchedule() {
llama_ctx_.reset();
}
-REGISTER_RESOURCE(RunLlamaCppInference, Processor);
-
} // namespace org::apache::nifi::minifi::extensions::llamacpp::processors
diff --git a/extensions/llamacpp/processors/RunLlamaCppInference.h b/extensions/llamacpp/processors/RunLlamaCppInference.h
index b82e196ed..76ae3a1f6 100644
--- a/extensions/llamacpp/processors/RunLlamaCppInference.h
+++ b/extensions/llamacpp/processors/RunLlamaCppInference.h
@@ -20,38 +20,23 @@
#include <mutex>
#include <atomic>
-#include "core/ProcessorImpl.h"
-#include "core/logging/LoggerFactory.h"
+#include "api/core/ProcessorImpl.h"
#include "core/PropertyDefinitionBuilder.h"
#include "LlamaContext.h"
-#include "minifi-cpp/core/ProcessorMetricsExtension.h"
-#include "core/state/Value.h"
+#include "api/utils/Export.h"
+#include "minifi-cpp/core/Annotation.h"
namespace org::apache::nifi::minifi::extensions::llamacpp::processors {
using LlamaContextProvider =
std::function<std::unique_ptr<LlamaContext>(const std::filesystem::path& model_path, const LlamaSamplerParams& llama_sampler_params, const LlamaContextParams& llama_ctx_params)>;
-class RunLlamaCppInferenceMetrics : public core::ProcessorMetricsExtension {
+class RunLlamaCppInferenceMetrics {
public:
- RunLlamaCppInferenceMetrics() = default;
-
- std::vector<state::response::SerializedResponseNode> serialize() override {
- std::vector<state::response::SerializedResponseNode> resp;
-
- state::response::SerializedResponseNode tokens_in_node{"TokensIn", tokens_in.load()};
- resp.push_back(tokens_in_node);
-
- state::response::SerializedResponseNode tokens_out_node{"TokensOut", tokens_out.load()};
- resp.push_back(tokens_out_node);
-
- return resp;
- }
-
- std::vector<state::PublishedMetric> calculateMetrics() override {
- std::vector<state::PublishedMetric> metrics;
- metrics.push_back({"tokens_in", static_cast<double>(tokens_in.load()), {}});
- metrics.push_back({"tokens_out", static_cast<double>(tokens_out.load()), {}});
+ minifi::api::core::PublishedMetrics calculateMetrics() const {
+ minifi::api::core::PublishedMetrics metrics;
+ metrics.push_back({"tokens_in", static_cast<double>(tokens_in.load())});
+ metrics.push_back({"tokens_out", static_cast<double>(tokens_out.load())});
return metrics;
}
@@ -59,13 +44,11 @@ class RunLlamaCppInferenceMetrics : public core::ProcessorMetricsExtension {
std::atomic<uint64_t> tokens_out{0};
};
-class RunLlamaCppInference : public core::ProcessorImpl {
+class RunLlamaCppInference : public api::core::ProcessorImpl {
public:
explicit RunLlamaCppInference(core::ProcessorMetadata metadata, LlamaContextProvider llama_context_provider = {})
- : core::ProcessorImpl(metadata),
- llama_context_provider_(std::move(llama_context_provider)) {
- metrics_extension_ = gsl::make_not_null(std::make_shared<RunLlamaCppInferenceMetrics>());
- }
+ : api::core::ProcessorImpl(metadata),
+ llama_context_provider_(std::move(llama_context_provider)) {}
~RunLlamaCppInference() override = default;
EXTENSIONAPI static constexpr const char* Description = "LlamaCpp processor to use llama.cpp library for running language model inference. "
@@ -174,12 +157,10 @@ class RunLlamaCppInference : public core::ProcessorImpl {
EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED;
EXTENSIONAPI static constexpr bool IsSingleThreaded = true;
- ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS
-
- void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override;
- void onTrigger(core::ProcessContext& context, core::ProcessSession& session) override;
- void initialize() override;
- void notifyStop() override;
+ MinifiStatus onScheduleImpl(api::core::ProcessContext& context) override;
+ MinifiStatus onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) override;
+ void onUnSchedule() override;
+ minifi::api::core::PublishedMetrics calculateMetrics() const override {return metrics_.calculateMetrics();}
private:
void increaseTokensIn(uint64_t token_count);
@@ -190,6 +171,8 @@ class RunLlamaCppInference : public core::ProcessorImpl {
LlamaContextProvider llama_context_provider_;
std::unique_ptr<LlamaContext> llama_ctx_;
+
+ RunLlamaCppInferenceMetrics metrics_;
};
} // namespace org::apache::nifi::minifi::extensions::llamacpp::processors
diff --git a/extensions/llamacpp/tests/CMakeLists.txt b/extensions/llamacpp/tests/CMakeLists.txt
index d1cd79e33..027dbc496 100644
--- a/extensions/llamacpp/tests/CMakeLists.txt
+++ b/extensions/llamacpp/tests/CMakeLists.txt
@@ -29,6 +29,7 @@ FOREACH(testfile ${LLAMACPP_TESTS})
target_link_libraries(${testfilename} Catch2WithMain)
target_link_libraries(${testfilename} minifi-llamacpp)
target_link_libraries(${testfilename} minifi-standard-processors)
+ target_link_libraries(${testfilename} libminifi-c-unittest)
target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata")
MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1")
diff --git a/extensions/llamacpp/tests/RunLlamaCppInferenceTests.cpp b/extensions/llamacpp/tests/RunLlamaCppInferenceTests.cpp
index 4cfe0a25e..ec507af33 100644
--- a/extensions/llamacpp/tests/RunLlamaCppInferenceTests.cpp
+++ b/extensions/llamacpp/tests/RunLlamaCppInferenceTests.cpp
@@ -15,12 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#include "unit/TestBase.h"
-#include "unit/Catch.h"
#include "RunLlamaCppInference.h"
-#include "unit/SingleProcessorTestController.h"
+#include "api/core/Resource.h"
#include "minifi-cpp/core/FlowFile.h"
-#include "unit/ProcessorUtils.h"
+#include "unit/Catch.h"
+#include "unit/SingleProcessorTestController.h"
+#include "unit/TestBase.h"
+#include "utils/CProcessor.h"
+#include "unit/TestUtils.h"
+#include "CProcessorTestUtils.h"
namespace org::apache::nifi::minifi::extensions::llamacpp::test {
@@ -79,7 +82,7 @@ TEST_CASE("Prompt is generated correctly with default parameters") {
std::filesystem::path test_model_path;
processors::LlamaSamplerParams test_sampler_params;
processors::LlamaContextParams test_context_params;
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path& model_path, const processors::LlamaSamplerParams& sampler_params, const processors::LlamaContextParams& context_params) {
test_model_path = model_path;
@@ -125,7 +128,7 @@ TEST_CASE("Prompt is generated correctly with custom parameters") {
std::filesystem::path test_model_path;
processors::LlamaSamplerParams test_sampler_params;
processors::LlamaContextParams test_context_params;
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path& model_path, const processors::LlamaSamplerParams& sampler_params, const processors::LlamaContextParams& context_params) {
test_model_path = model_path;
@@ -177,7 +180,7 @@ TEST_CASE("Prompt is generated correctly with custom parameters") {
TEST_CASE("Empty flow file does not include input data in prompt") {
auto mock_llama_context = std::make_unique<MockLlamaContext>();
auto mock_llama_context_ptr = mock_llama_context.get();
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams&, const processors::LlamaContextParams&) {
return std::move(mock_llama_context);
@@ -201,7 +204,7 @@ TEST_CASE("Empty flow file does not include input data in prompt") {
}
TEST_CASE("Invalid values for optional double type properties throw exception") {
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams&, const processors::LlamaContextParams&) {
return std::make_unique<MockLlamaContext>();
@@ -224,13 +227,14 @@ TEST_CASE("Invalid values for optional double type properties throw exception")
property_name = processors::RunLlamaCppInference::MinP.name;
}
- REQUIRE_THROWS_WITH(controller.trigger(minifi::test::InputFlowFileData{.content = "42", .attributes = {}}),
- fmt::format("Expected parsable float from RunLlamaCppInference::{}, but got GeneralParsingError (Parsing Error:0)", property_name));
+ REQUIRE_THROWS(controller.trigger(minifi::test::InputFlowFileData{.content = "42", .attributes = {}}));
+ CHECK(minifi::test::utils::verifyLogLinePresenceInPollTime(1s,
+ fmt::format("Expected parsable float from RunLlamaCppInference::{}, but got GeneralParsingError (Parsing Error:0)", property_name)));
}
TEST_CASE("Top K property empty and invalid values are handled properly") {
std::optional<int32_t> test_top_k = 0;
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams& sampler_params, const processors::LlamaContextParams&) {
test_top_k = sampler_params.top_k;
@@ -246,8 +250,9 @@ TEST_CASE("Top K property empty and invalid values are handled properly") {
}
SECTION("Invalid value for Top K property") {
REQUIRE(controller.getProcessor()->setProperty(processors::RunLlamaCppInference::TopK.name, "invalid_value"));
- REQUIRE_THROWS_WITH(controller.trigger(minifi::test::InputFlowFileData{.content = "42", .attributes = {}}),
- "Expected parsable int64_t from \"RunLlamaCppInference::Top K\", but got GeneralParsingError (Parsing Error:0)");
+ REQUIRE_THROWS(controller.trigger(minifi::test::InputFlowFileData{.content = "42", .attributes = {}}));
+ CHECK(minifi::test::utils::verifyLogLinePresenceInPollTime(1s,
+ "Expected parsable int64_t from \"RunLlamaCppInference::Top K\", but got GeneralParsingError (Parsing Error:0)"));
}
}
@@ -262,7 +267,7 @@ TEST_CASE("Error handling during generation and applying template") {
mock_llama_context->setApplyTemplateFailure();
}
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams&, const processors::LlamaContextParams&) {
return std::move(mock_llama_context);
@@ -280,7 +285,7 @@ TEST_CASE("Error handling during generation and applying template") {
}
TEST_CASE("Route flow file to failure when prompt and input data is empty") {
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams&, const processors::LlamaContextParams&) {
return std::make_unique<MockLlamaContext>();
@@ -300,7 +305,7 @@ TEST_CASE("Route flow file to failure when prompt and input data is empty") {
TEST_CASE("System prompt is optional") {
auto mock_llama_context = std::make_unique<MockLlamaContext>();
auto mock_llama_context_ptr = mock_llama_context.get();
- minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ minifi::test::SingleProcessorTestController controller(minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams&, const processors::LlamaContextParams&) {
return std::move(mock_llama_context);
@@ -322,7 +327,7 @@ TEST_CASE("System prompt is optional") {
}
TEST_CASE("Test output metrics") {
- auto processor = minifi::test::utils::make_custom_processor<processors::RunLlamaCppInference>(
+ auto processor = minifi::test::utils::make_custom_c_processor<processors::RunLlamaCppInference>(
core::ProcessorMetadata{utils::Identifier{}, "RunLlamaCppInference", logging::LoggerFactory<processors::RunLlamaCppInference>::getLogger()},
[&](const std::filesystem::path&, const processors::LlamaSamplerParams&, const processors::LlamaContextParams&) {
return std::make_unique<MockLlamaContext>();
diff --git a/extensions/lua/LuaScriptFlowFile.cpp b/extensions/lua/LuaScriptFlowFile.cpp
index 6476adde3..4eb3196d9 100644
--- a/extensions/lua/LuaScriptFlowFile.cpp
+++ b/extensions/lua/LuaScriptFlowFile.cpp
@@ -48,12 +48,12 @@ bool LuaScriptFlowFile::addAttribute(const std::string& key, const std::string&
return flow_file_->addAttribute(key, value);
}
-bool LuaScriptFlowFile::setAttribute(const std::string& key, const std::string& value) {
+void LuaScriptFlowFile::setAttribute(const std::string& key, const std::string& value) {
if (!flow_file_) {
throw std::runtime_error("Access of FlowFile after it has been released");
}
- return flow_file_->setAttribute(key, value);
+ flow_file_->setAttribute(key, value);
}
bool LuaScriptFlowFile::updateAttribute(std::string_view key, const std::string& value) {
diff --git a/extensions/lua/LuaScriptFlowFile.h b/extensions/lua/LuaScriptFlowFile.h
index 477f23a0f..5e9af5857 100644
--- a/extensions/lua/LuaScriptFlowFile.h
+++ b/extensions/lua/LuaScriptFlowFile.h
@@ -30,7 +30,7 @@ class LuaScriptFlowFile {
public:
explicit LuaScriptFlowFile(std::shared_ptr<core::FlowFile> flow_file);
std::string getAttribute(const std::string &key);
- bool setAttribute(const std::string &key, const std::string &value);
+ void setAttribute(const std::string &key, const std::string &value);
bool addAttribute(const std::string &key, const std::string &value);
bool updateAttribute(std::string_view key, const std::string& value);
bool removeAttribute(std::string_view key);
diff --git a/extensions/opencv/OpenCVLoader.cpp b/extensions/opencv/OpenCVLoader.cpp
index ffb313f34..79a437f75 100644
--- a/extensions/opencv/OpenCVLoader.cpp
+++ b/extensions/opencv/OpenCVLoader.cpp
@@ -19,7 +19,7 @@
#include "utils/Environment.h"
#include "minifi-cpp/agent/agent_version.h"
#include "minifi-c/minifi-c.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
#include "core/Resource.h"
namespace minifi = org::apache::nifi::minifi;
@@ -38,7 +38,9 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
.name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
.version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
.deinit = nullptr,
- .user_data = nullptr
+ .user_data = nullptr,
+ .processors_count = 0,
+ .processors_ptr = nullptr
};
- return MinifiCreateExtension(&ext_create_info);
+ return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/python/ExecutePythonProcessor.cpp b/extensions/python/ExecutePythonProcessor.cpp
index 33480ab8f..2d4d76157 100644
--- a/extensions/python/ExecutePythonProcessor.cpp
+++ b/extensions/python/ExecutePythonProcessor.cpp
@@ -38,9 +38,8 @@ namespace org::apache::nifi::minifi::extensions::python::processors {
void ExecutePythonProcessor::initialize() {
initializeScript();
- std::vector<core::PropertyReference> all_properties;
- ranges::transform(python_properties_, std::back_inserter(all_properties), &core::Property::getReference);
- setSupportedProperties(all_properties);
+ // only include supported properties that are defined in the python processor
+ setSupportedProperties(gsl::make_span(python_properties_));
setSupportedRelationships(Relationships);
logger_->log_debug("Processor has been initialized.");
}
@@ -68,10 +67,10 @@ std::vector<core::Relationship> ExecutePythonProcessor::getPythonRelationships()
return relationships;
}
-void ExecutePythonProcessor::setLoggerCallback(const std::function<void(core::logging::LOG_LEVEL level, const std::string& message)>& callback) {
+void ExecutePythonProcessor::forEachLogger(const std::function<void(std::shared_ptr<core::logging::Logger>)>& callback) {
gsl_Expects(logger_ && python_logger_);
- logger_->setLogCallback(callback);
- python_logger_->setLogCallback(callback);
+ callback(logger_);
+ callback(python_logger_);
}
void ExecutePythonProcessor::initializeScript() {
diff --git a/extensions/python/ExecutePythonProcessor.h b/extensions/python/ExecutePythonProcessor.h
index f16799d03..a237e19bd 100644
--- a/extensions/python/ExecutePythonProcessor.h
+++ b/extensions/python/ExecutePythonProcessor.h
@@ -116,7 +116,7 @@ class ExecutePythonProcessor : public core::ProcessorImpl {
}
std::vector<core::Relationship> getPythonRelationships() const;
- void setLoggerCallback(const std::function<void(core::logging::LOG_LEVEL level, const std::string& message)>& callback) override;
+ void forEachLogger(const std::function<void(std::shared_ptr<core::logging::Logger>)>& callback) override;
private:
mutable std::mutex python_properties_mutex_;
diff --git a/extensions/python/PythonCreator.h b/extensions/python/PythonCreator.h
index db93e01b7..17dbce119 100644
--- a/extensions/python/PythonCreator.h
+++ b/extensions/python/PythonCreator.h
@@ -41,8 +41,7 @@
#include "utils/file/FilePattern.h"
#include "utils/file/FileUtils.h"
#include "minifi-cpp/core/ProcessorDescriptor.h"
-#include "minifi-c/minifi-c.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
namespace org::apache::nifi::minifi::extensions::python {
@@ -52,14 +51,12 @@ class DummyProcessorDescriptor : public core::ProcessorDescriptor {
void setSupportedRelationships(std::span<const core::RelationshipDefinition> /*relationships*/) override {}
void setSupportedProperties(std::span<const core::PropertyReference> /*properties*/) override {}
+ void setSupportedProperties(std::span<const core::Property> /*properties*/) override {}
};
class DummyLogger : public core::logging::Logger {
public:
void set_max_log_size(int /*size*/) override {}
- std::optional<std::string> get_id() override {
- return std::nullopt;
- }
void log_string(core::logging::LOG_LEVEL /*level*/, std::string /*str*/) override {}
bool should_log(core::logging::LOG_LEVEL /*level*/) override {
return false;
@@ -68,14 +65,6 @@ class DummyLogger : public core::logging::Logger {
return core::logging::LOG_LEVEL::off;
}
- int getMaxLogSize() override {
- return 0;
- }
-
- void setLogCallback(const std::function<void(core::logging::LOG_LEVEL level, const std::string&)>& /*callback*/) override {
- // pass
- }
-
~DummyLogger() override = default;
};
diff --git a/extensions/python/PythonDependencyInstaller.cpp b/extensions/python/PythonDependencyInstaller.cpp
index bbc9076d3..f3b92eece 100644
--- a/extensions/python/PythonDependencyInstaller.cpp
+++ b/extensions/python/PythonDependencyInstaller.cpp
@@ -25,7 +25,7 @@
#include "types/Types.h"
#include "utils/OptionalUtils.h"
#include "utils/ConfigurationUtils.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
namespace org::apache::nifi::minifi::extensions::python {
diff --git a/extensions/python/PythonDependencyInstaller.h b/extensions/python/PythonDependencyInstaller.h
index 7b2e2c45b..fc20f06f3 100644
--- a/extensions/python/PythonDependencyInstaller.h
+++ b/extensions/python/PythonDependencyInstaller.h
@@ -22,7 +22,7 @@
#include "minifi-cpp/core/logging/Logger.h"
#include "core/logging/LoggerFactory.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
namespace org::apache::nifi::minifi::extensions::python {
diff --git a/extensions/python/pythonlibloader/PythonLibLoader.cpp b/extensions/python/pythonlibloader/PythonLibLoader.cpp
index 81f047f0c..398322f8f 100644
--- a/extensions/python/pythonlibloader/PythonLibLoader.cpp
+++ b/extensions/python/pythonlibloader/PythonLibLoader.cpp
@@ -24,7 +24,7 @@
#include "core/logging/LoggerFactory.h"
#include "minifi-cpp/agent/agent_version.h"
#include "minifi-c/minifi-c.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
#include "core/Resource.h"
#if defined(WIN32)
@@ -101,7 +101,7 @@ class PythonLibLoader {
extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
static PythonLibLoader python_lib_loader([&] (std::string_view key) -> std::optional<std::string> {
std::optional<std::string> result;
- MinifiConfigureGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
+ MinifiConfigGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
*static_cast<std::optional<std::string>*>(user_data) = std::string{value.data, value.length};
}, &result);
return result;
@@ -110,7 +110,9 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
.name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
.version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
.deinit = nullptr,
- .user_data = nullptr
+ .user_data = nullptr,
+ .processors_count = 0,
+ .processors_ptr = nullptr
};
- return MinifiCreateExtension(&ext_create_info);
+ return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/python/pythonloader/PyProcLoader.cpp b/extensions/python/pythonloader/PyProcLoader.cpp
index 7df2532af..3778f15ec 100644
--- a/extensions/python/pythonloader/PyProcLoader.cpp
+++ b/extensions/python/pythonloader/PyProcLoader.cpp
@@ -16,7 +16,7 @@
* limitations under the License.
*/
#include "minifi-c/minifi-c.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
#include "PythonCreator.h"
#include "minifi-cpp/agent/agent_version.h"
#include "PythonBindings.h"
@@ -36,7 +36,7 @@ extern "C" const int LOAD_MODULE_AS_GLOBAL = 1;
extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
getPythonCreator().configure([&] (std::string_view key) -> std::optional<std::string> {
std::optional<std::string> result;
- MinifiConfigureGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
+ MinifiConfigGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
*static_cast<std::optional<std::string>*>(user_data) = std::string{value.data, value.length};
}, &result);
return result;
@@ -45,7 +45,9 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
.name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
.version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
.deinit = nullptr,
- .user_data = nullptr
+ .user_data = nullptr,
+ .processors_count = 0,
+ .processors_ptr = nullptr
};
- return MinifiCreateExtension(&ext_create_info);
+ return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/python/types/PyScriptFlowFile.cpp b/extensions/python/types/PyScriptFlowFile.cpp
index 178e422a1..832e09419 100644
--- a/extensions/python/types/PyScriptFlowFile.cpp
+++ b/extensions/python/types/PyScriptFlowFile.cpp
@@ -135,7 +135,8 @@ PyObject* PyScriptFlowFile::setAttribute(PyScriptFlowFile* self, PyObject* args)
return nullptr;
}
- return object::returnReference(flow_file->setAttribute(key, value));
+ flow_file->setAttribute(key, value);
+ Py_RETURN_NONE;
}
PyObject* PyScriptFlowFile::getSize(PyScriptFlowFile* self, PyObject* /*args*/) {
diff --git a/extensions/sftp/SFTPLoader.cpp b/extensions/sftp/SFTPLoader.cpp
index 40bc2db8c..8a67842de 100644
--- a/extensions/sftp/SFTPLoader.cpp
+++ b/extensions/sftp/SFTPLoader.cpp
@@ -20,7 +20,7 @@
#include "minifi-cpp/properties/Configure.h"
#include "client/SFTPClient.h"
#include "minifi-c/minifi-c.h"
-#include "utils/minifi-c-utils.h"
+#include "utils/ExtensionInitUtils.h"
#include "core/Resource.h"
namespace minifi = org::apache::nifi::minifi;
@@ -40,7 +40,9 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
curl_global_cleanup();
libssh2_exit();
},
- .user_data = nullptr
+ .user_data = nullptr,
+ .processors_count = 0,
+ .processors_ptr = nullptr
};
- return MinifiCreateExtension(&ext_create_info);
+ return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/sql/processors/QueryDatabaseTable.h b/extensions/sql/processors/QueryDatabaseTable.h
index c148fe974..e8eb9dec0 100644
--- a/extensions/sql/processors/QueryDatabaseTable.h
+++ b/extensions/sql/processors/QueryDatabaseTable.h
@@ -97,7 +97,7 @@ class QueryDatabaseTable: public SQLProcessor, public FlowFileSource {
EXTENSIONAPI static constexpr auto Relationships = std::array{Success};
EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true;
- EXTENSIONAPI static constexpr auto InitialMaxValue = core::DynamicProperty{"initial.maxvalue.<max_value_column>",
+ EXTENSIONAPI static constexpr auto InitialMaxValue = core::DynamicPropertyDefinition{"initial.maxvalue.<max_value_column>",
"Initial maximum value for the specified column",
"Specifies an initial max value for max value column(s). Properties should be added in the format `initial.maxvalue.<max_value_column>`. "
"This value is only used the first time the table is accessed (when a Maximum Value Column is specified).",
diff --git a/extensions/standard-processors/processors/EvaluateJsonPath.h b/extensions/standard-processors/processors/EvaluateJsonPath.h
index 681a4b3b6..7ed248d78 100644
--- a/extensions/standard-processors/processors/EvaluateJsonPath.h
+++ b/extensions/standard-processors/processors/EvaluateJsonPath.h
@@ -21,6 +21,7 @@
#include <array>
#include "core/ProcessorImpl.h"
+#include "minifi-cpp/core/PropertyDefinition.h"
#include "core/PropertyDefinitionBuilder.h"
#include "minifi-cpp/core/PropertyValidator.h"
#include "minifi-cpp/core/RelationshipDefinition.h"
@@ -163,7 +164,7 @@ class EvaluateJsonPath final : public core::ProcessorImpl {
EXTENSIONAPI static constexpr auto Relationships = std::array{Failure, Matched, Unmatched};
EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true;
- EXTENSIONAPI static constexpr auto EvaluationResult = core::DynamicProperty{"Evaluation Result", "JsonPath expression to evaluate", "Dynamic property values are evaluated as JsonPaths. "
+ EXTENSIONAPI static constexpr auto EvaluationResult = core::DynamicPropertyDefinition{"Evaluation Result", "JsonPath expression to evaluate", "Dynamic property values are evaluated as JsonPaths. "
"In case of 'flowfile-content' destination, only one dynamic property with JsonPath may be specified, in this case the name of the property is ignored. "
"In case of 'flowfile-attribute' destination, the result of the JsonPath evaluation is written to the attribute matching the dynamic property name.", true};
EXTENSIONAPI static constexpr auto DynamicProperties = std::array{EvaluationResult};
diff --git a/extensions/standard-processors/processors/RetryFlowFile.h b/extensions/standard-processors/processors/RetryFlowFile.h
index b144af28c..43f645104 100644
--- a/extensions/standard-processors/processors/RetryFlowFile.h
+++ b/extensions/standard-processors/processors/RetryFlowFile.h
@@ -128,7 +128,7 @@ class RetryFlowFile : public core::ProcessorImpl {
};
EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true;
- EXTENSIONAPI static constexpr auto RetriesExceededAttribute = core::DynamicProperty{"Exceeded FlowFile Attribute Key",
+ EXTENSIONAPI static constexpr auto RetriesExceededAttribute = core::DynamicPropertyDefinition{"Exceeded FlowFile Attribute Key",
"The value of the attribute added to the FlowFile",
"One or more dynamic properties can be used to add attributes to FlowFiles passed to the 'retries_exceeded' relationship.",
true};
diff --git a/extensions/standard-processors/processors/RouteText.h b/extensions/standard-processors/processors/RouteText.h
index 688cbda55..f65f37c71 100644
--- a/extensions/standard-processors/processors/RouteText.h
+++ b/extensions/standard-processors/processors/RouteText.h
@@ -193,7 +193,7 @@ class RouteText : public core::ProcessorImpl {
EXTENSIONAPI static constexpr auto OutputAttributes = std::array<core::OutputAttributeReference, 1>{Group};
EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true;
- EXTENSIONAPI static constexpr auto RelationshipToRouteTo = core::DynamicProperty{"Relationship Name",
+ EXTENSIONAPI static constexpr auto RelationshipToRouteTo = core::DynamicPropertyDefinition{"Relationship Name",
"value to match against",
"Routes data that matches the value specified in the Dynamic Property Value to the Relationship specified in the Dynamic Property Key.",
true};
diff --git a/extensions/standard-processors/processors/SplitJson.h b/extensions/standard-processors/processors/SplitJson.h
index 289ef16a6..8eaf65dd5 100644
--- a/extensions/standard-processors/processors/SplitJson.h
+++ b/extensions/standard-processors/processors/SplitJson.h
@@ -22,6 +22,7 @@
#include <optional>
#include "core/ProcessorImpl.h"
+#include "minifi-cpp/core/PropertyDefinition.h"
#include "core/PropertyDefinitionBuilder.h"
#include "minifi-cpp/core/PropertyValidator.h"
#include "minifi-cpp/core/RelationshipDefinition.h"
diff --git a/extensions/standard-processors/tests/integration/TailFileIntegrationTest.cpp b/extensions/standard-processors/tests/integration/TailFileIntegrationTest.cpp
index fbf241c2a..2b4186b10 100644
--- a/extensions/standard-processors/tests/integration/TailFileIntegrationTest.cpp
+++ b/extensions/standard-processors/tests/integration/TailFileIntegrationTest.cpp
@@ -29,6 +29,7 @@
#include "unit/TestUtils.h"
#include "unit/Catch.h"
#include "utils/TimeUtil.h"
+#include "utils/TimeZoneUtils.h"
using namespace std::literals::chrono_literals;
diff --git a/libminifi/include/controllers/NetworkPrioritizerService.h b/libminifi/include/controllers/NetworkPrioritizerService.h
index fcb08c10a..9253659c8 100644
--- a/libminifi/include/controllers/NetworkPrioritizerService.h
+++ b/libminifi/include/controllers/NetworkPrioritizerService.h
@@ -32,6 +32,7 @@
#include "minifi-cpp/core/PropertyValidator.h"
#include "io/NetworkPrioritizer.h"
#include "minifi-cpp/utils/Export.h"
+#include "minifi-cpp/utils/TimeUtil.h"
namespace org::apache::nifi::minifi::controllers {
diff --git a/libminifi/include/core/FlowFile.h b/libminifi/include/core/FlowFile.h
index 2c2b3c583..72f1aea2f 100644
--- a/libminifi/include/core/FlowFile.h
+++ b/libminifi/include/core/FlowFile.h
@@ -156,8 +156,8 @@ class FlowFileImpl : public CoreComponentImpl, public ReferenceContainerImpl, pu
/**
* setAttribute, if attribute already there, update it, else, add it
*/
- bool setAttribute(std::string_view key, std::string value) override {
- return attributes_.insert_or_assign(std::string{key}, std::move(value)).second;
+ void setAttribute(std::string_view key, std::string value) override {
+ attributes_.insert_or_assign(std::string{key}, std::move(value));
}
/**
diff --git a/libminifi/include/core/ProcessSession.h b/libminifi/include/core/ProcessSession.h
index d176bb3b2..cc5e2c964 100644
--- a/libminifi/include/core/ProcessSession.h
+++ b/libminifi/include/core/ProcessSession.h
@@ -72,7 +72,6 @@ class ProcessSessionImpl : public ReferenceContainerImpl, public virtual Process
std::shared_ptr<core::FlowFile> clone(const core::FlowFile& parent) override;
std::shared_ptr<core::FlowFile> clone(const core::FlowFile& parent, int64_t offset, int64_t size) override;
- void transfer(const std::shared_ptr<core::FlowFile>& flow, const Relationship& relationship) override;
void transferToCustomRelationship(const std::shared_ptr<core::FlowFile>& flow, const std::string& relationship_name) override;
void putAttribute(core::FlowFile& flow, std::string_view key, const std::string& value) override;
@@ -128,6 +127,7 @@ class ProcessSessionImpl : public ReferenceContainerImpl, public virtual Process
}
bool hasBeenTransferred(const core::FlowFile &flow) const override;
+ void transfer(const std::shared_ptr<core::FlowFile>& flow, const Relationship& relationship) override;
ProcessSessionImpl(const ProcessSessionImpl &parent) = delete;
ProcessSessionImpl &operator=(const ProcessSessionImpl &parent) = delete;
diff --git a/libminifi/include/core/Processor.h b/libminifi/include/core/Processor.h
index 7069854a5..5cb2ed9ce 100644
--- a/libminifi/include/core/Processor.h
+++ b/libminifi/include/core/Processor.h
@@ -35,7 +35,7 @@
#include "minifi-cpp/core/Property.h"
#include "core/Core.h"
#include "minifi-cpp/core/Annotation.h"
-#include "minifi-cpp/core/DynamicProperty.h"
+#include "minifi-cpp/core/DynamicPropertyDefinition.h"
#include "minifi-cpp/core/Scheduling.h"
#include "minifi-cpp/core/state/nodes/MetricsBase.h"
#include "core/ProcessorMetrics.h"
@@ -123,7 +123,7 @@ class Processor : public ConnectableImpl, public ConfigurableComponentImpl, publ
void setLoggerCallback(const std::function<void(logging::LOG_LEVEL level, const std::string& message)>& callback);
void restore(const std::shared_ptr<FlowFile>& file) override;
- static constexpr auto DynamicProperties = std::array<DynamicProperty, 0>{};
+ static constexpr auto DynamicProperties = std::array<DynamicPropertyDefinition, 0>{};
static constexpr auto OutputAttributes = std::array<OutputAttributeReference, 0>{};
diff --git a/libminifi/include/core/extension/Utils.h b/libminifi/include/core/extension/Utils.h
index d6e5e11bb..520827193 100644
--- a/libminifi/include/core/extension/Utils.h
+++ b/libminifi/include/core/extension/Utils.h
@@ -29,7 +29,6 @@
#include "minifi-cpp/utils/gsl.h"
#include "utils/StringUtils.h"
#include "utils/file/FileUtils.h"
-#include "utils/file/PathUtils.h"
namespace org::apache::nifi::minifi::core::extension::internal {
@@ -45,24 +44,35 @@ class Timer {
Callback cb_;
};
+enum LibraryType {
+ Cpp,
+ CApi,
+ Invalid
+};
+
struct LibraryDescriptor {
std::string name;
std::filesystem::path dir;
std::string filename;
+
+ [[nodiscard]] bool verify_as_cpp_extension() const;
+
+ [[nodiscard]] bool verify_as_c_extension(const std::shared_ptr<logging::Logger>& logger) const;
+
[[nodiscard]]
- bool verify(const std::shared_ptr<logging::Logger>& logger) const {
+ LibraryType verify(const std::shared_ptr<logging::Logger>& logger) const {
const auto path = getFullPath();
- if (!std::filesystem::exists(path)) {
- throw std::runtime_error{"File not found: " + path.string()};
- }
const Timer timer{[&](const std::chrono::milliseconds elapsed) {
logger->log_debug("Verification for '{}' took {}", path, elapsed);
}};
- const std::string_view begin_marker = "__EXTENSION_BUILD_IDENTIFIER_BEGIN__";
- const std::string_view end_marker = "__EXTENSION_BUILD_IDENTIFIER_END__";
- const std::string magic_constant = utils::string::join_pack(begin_marker, AgentBuild::BUILD_IDENTIFIER, end_marker);
- return utils::file::contains(path, magic_constant);
+ if (verify_as_cpp_extension()) {
+ return Cpp;
+ }
+ if (verify_as_c_extension(logger)) {
+ return CApi;
+ }
+ return Invalid;
}
[[nodiscard]]
@@ -71,7 +81,7 @@ struct LibraryDescriptor {
}
};
-std::optional<LibraryDescriptor> asDynamicLibrary(const std::filesystem::path& path) {
+inline std::optional<LibraryDescriptor> asDynamicLibrary(const std::filesystem::path& path) {
#if defined(WIN32)
static const std::string_view extension = ".dll";
#elif defined(__APPLE__)
diff --git a/libminifi/include/core/logging/LoggerBase.h b/libminifi/include/core/logging/LoggerBase.h
index d9a4c4815..175867f49 100644
--- a/libminifi/include/core/logging/LoggerBase.h
+++ b/libminifi/include/core/logging/LoggerBase.h
@@ -120,17 +120,21 @@ class LoggerBase : public Logger {
max_log_size_ = size;
}
+ virtual std::optional<std::string> get_id() = 0;
bool should_log(LOG_LEVEL level) override;
void log_string(LOG_LEVEL level, std::string str) override;
LOG_LEVEL level() const override;
- void setLogCallback(const std::function<void(LOG_LEVEL level, const std::string&)>& callback) override;
+ void setLogCallback(const std::function<void(LOG_LEVEL level, const std::string&)>& callback);
+
+ private:
+ std::string trimToMaxSizeAndAddId(std::string my_string);
protected:
LoggerBase(std::shared_ptr<spdlog::logger> delegate, std::shared_ptr<LoggerControl> controller);
LoggerBase(std::shared_ptr<spdlog::logger> delegate); // NOLINT
- int getMaxLogSize() override {
+ int getMaxLogSize() {
return max_log_size_.load();
}
diff --git a/libminifi/include/core/logging/alert/AlertSink.h b/libminifi/include/core/logging/alert/AlertSink.h
index 9487a61e3..b251e378c 100644
--- a/libminifi/include/core/logging/alert/AlertSink.h
+++ b/libminifi/include/core/logging/alert/AlertSink.h
@@ -32,6 +32,7 @@
#include "properties/Configure.h"
#include "spdlog/sinks/base_sink.h"
#include "minifi-cpp/controllers/SSLContextServiceInterface.h"
+#include "minifi-cpp/utils/TimeUtil.h"
namespace org::apache::nifi::minifi::controllers {
class SSLContextServiceInterface;
diff --git a/libminifi/include/utils/CProcessor.h b/libminifi/include/utils/CProcessor.h
new file mode 100644
index 000000000..bef73d9e9
--- /dev/null
+++ b/libminifi/include/utils/CProcessor.h
@@ -0,0 +1,176 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "minifi-cpp/core/Annotation.h"
+#include "core/ProcessorMetrics.h"
+#include "minifi-c/minifi-c.h"
+#include "minifi-cpp/agent/agent_docs.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "minifi-cpp/core/ProcessorApi.h"
+#include "minifi-cpp/core/ProcessorDescriptor.h"
+#include "minifi-cpp/core/ProcessorMetadata.h"
+
+namespace org::apache::nifi::minifi::utils {
+
+class CProcessor;
+
+class CProcessorMetricsWrapper : public minifi::core::ProcessorMetricsExtension {
+ public:
+ explicit CProcessorMetricsWrapper(const CProcessor& source_processor)
+ : source_processor_(source_processor) {
+ }
+
+ std::vector<minifi::state::response::SerializedResponseNode> serialize() override;
+
+ std::vector<minifi::state::PublishedMetric> calculateMetrics() override;
+
+ private:
+ const CProcessor& source_processor_;
+};
+
+struct CProcessorClassDescription {
+ std::string name;
+ std::vector<minifi::core::Property> class_properties;
+ std::vector<minifi::core::Relationship> class_relationships;
+ bool supports_dynamic_properties;
+ bool supports_dynamic_relationships;
+ minifi::core::annotation::Input input_requirement;
+ bool is_single_threaded;
+
+ MinifiProcessorCallbacks callbacks;
+};
+
+class CProcessor : public minifi::core::ProcessorApi {
+ public:
+ CProcessor(CProcessorClassDescription class_description, minifi::core::ProcessorMetadata metadata)
+ : class_description_(std::move(class_description)),
+ metrics_extension_(std::make_shared<CProcessorMetricsWrapper>(*this)) {
+ metadata_ = metadata;
+ MinifiProcessorMetadata c_metadata;
+ auto uuid_str = metadata.uuid.to_string();
+ c_metadata.uuid = MinifiStringView{.data = uuid_str.data(), .length = uuid_str.length()};
+ c_metadata.name = MinifiStringView{.data = metadata.name.data(), .length = metadata.name.length()};
+ c_metadata.logger = reinterpret_cast<MinifiLogger*>(&metadata_.logger);
+ impl_ = class_description_.callbacks.create(c_metadata);
+ }
+ CProcessor(CProcessorClassDescription class_description, minifi::core::ProcessorMetadata metadata, gsl::owner<void*> impl)
+ : class_description_(std::move(class_description)),
+ impl_(impl),
+ metadata_(metadata),
+ metrics_extension_(std::make_shared<CProcessorMetricsWrapper>(*this)) {}
+ ~CProcessor() override {
+ class_description_.callbacks.destroy(impl_);
+ }
+
+ bool isWorkAvailable() override {
+ return static_cast<bool>(class_description_.callbacks.isWorkAvailable(impl_));
+ }
+
+ void restore(const std::shared_ptr<minifi::core::FlowFile>& file) override {
+ class_description_.callbacks.restore(impl_, reinterpret_cast<MINIFI_OWNED MinifiFlowFile*>(new std::shared_ptr<minifi::core::FlowFile>(file)));
+ }
+
+ bool supportsDynamicProperties() const override {
+ return class_description_.supports_dynamic_properties;
+ }
+
+ bool supportsDynamicRelationships() const override {
+ return class_description_.supports_dynamic_relationships;
+ }
+
+ void initialize(minifi::core::ProcessorDescriptor& descriptor) override {
+ descriptor.setSupportedProperties(std::span(class_description_.class_properties));
+
+ std::vector<minifi::core::RelationshipDefinition> relationships;
+ for (auto& rel : class_description_.class_relationships) {relationships.push_back(rel.getDefinition());}
+ descriptor.setSupportedRelationships(relationships);
+ }
+
+ bool isSingleThreaded() const override {
+ return class_description_.is_single_threaded;
+ }
+
+ std::string getProcessorType() const override {
+ return class_description_.name;
+ }
+
+ bool getTriggerWhenEmpty() const override {
+ return static_cast<bool>(class_description_.callbacks.getTriggerWhenEmpty(impl_));
+ }
+
+ void onTrigger(minifi::core::ProcessContext& process_context, minifi::core::ProcessSession& process_session) override {
+ std::optional<std::string> error;
+ auto status = class_description_.callbacks.onTrigger(impl_, reinterpret_cast<MinifiProcessContext*>(&process_context), reinterpret_cast<MinifiProcessSession*>(&process_session));
+ if (status == MINIFI_STATUS_PROCESSOR_YIELD) {
+ process_context.yield();
+ return;
+ }
+ if (status != MINIFI_STATUS_SUCCESS) {
+ throw minifi::Exception(minifi::ExceptionType::PROCESSOR_EXCEPTION, "Error while triggering processor");
+ }
+ }
+
+ void onSchedule(minifi::core::ProcessContext& process_context, minifi::core::ProcessSessionFactory& /*process_session_factory*/) override {
+ std::optional<std::string> error;
+ auto status = class_description_.callbacks.onSchedule(impl_, reinterpret_cast<MinifiProcessContext*>(&process_context));
+ if (status != MINIFI_STATUS_SUCCESS) {
+ throw minifi::Exception(minifi::ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Error while scheduling processor");
+ }
+ }
+
+ void onUnSchedule() override {
+ class_description_.callbacks.onUnSchedule(impl_);
+ }
+
+ void notifyStop() override {
+ class_description_.callbacks.onUnSchedule(impl_);
+ }
+
+ minifi::core::annotation::Input getInputRequirement() const override {
+ return class_description_.input_requirement;
+ }
+
+ std::shared_ptr<minifi::core::ProcessorMetricsExtension> getMetricsExtension() const override {
+ return metrics_extension_;
+ }
+
+ std::vector<minifi::state::PublishedMetric> getCustomMetrics() const;
+
+ void forEachLogger(const std::function<void(std::shared_ptr<minifi::core::logging::Logger>)>&) override {}
+
+ std::string getName() const {
+ return metadata_.name;
+ }
+
+ minifi::utils::Identifier getUUID() const {
+ return metadata_.uuid;
+ }
+
+ private:
+ CProcessorClassDescription class_description_;
+ gsl::owner<void*> impl_;
+ minifi::core::ProcessorMetadata metadata_;
+ gsl::not_null<std::shared_ptr<minifi::core::ProcessorMetricsExtension>> metrics_extension_;
+};
+
+void useCProcessorClassDescription(const MinifiProcessorClassDefinition& class_description, const std::function<void(ClassDescription, CProcessorClassDescription)>& fn);
+
+} // namespace org::apache::nifi::minifi::utils
diff --git a/libminifi/include/utils/FlowFileQueue.h b/libminifi/include/utils/FlowFileQueue.h
index df81982ea..c69d90d23 100644
--- a/libminifi/include/utils/FlowFileQueue.h
+++ b/libminifi/include/utils/FlowFileQueue.h
@@ -24,8 +24,7 @@
#include "minifi-cpp/core/FlowFile.h"
#include "MinMaxHeap.h"
#include "minifi-cpp/SwapManager.h"
-#include "utils/TimeUtil.h"
-#include "minifi-cpp/core/logging/Logger.h"
+#include "minifi-cpp/utils/TimeUtil.h"
namespace org::apache::nifi::minifi::test::utils {
struct FlowFileQueueTestAccessor;
diff --git a/libminifi/src/core/ProcessSession.cpp b/libminifi/src/core/ProcessSession.cpp
index a0d7bf884..576ed3a91 100644
--- a/libminifi/src/core/ProcessSession.cpp
+++ b/libminifi/src/core/ProcessSession.cpp
@@ -223,6 +223,7 @@ void ProcessSessionImpl::penalize(const std::shared_ptr<core::FlowFile> &flow) {
}
void ProcessSessionImpl::transfer(const std::shared_ptr<core::FlowFile>& flow, const Relationship& relationship) {
+ gsl_Expects(flow);
logger_->log_debug("Transferring {} from {} to relationship {}", flow->getUUIDStr(), process_context_->getProcessor().getName(), relationship.getName());
utils::Identifier uuid = flow->getUUID();
if (auto it = added_flowfiles_.find(uuid); it != added_flowfiles_.end()) {
diff --git a/libminifi/src/core/Processor.cpp b/libminifi/src/core/Processor.cpp
index 5c0da6f2a..72fca50a6 100644
--- a/libminifi/src/core/Processor.cpp
+++ b/libminifi/src/core/Processor.cpp
@@ -30,6 +30,7 @@
#include "minifi-cpp/Connection.h"
#include "core/Connectable.h"
#include "core/logging/LoggerFactory.h"
+#include "core/logging/LoggerBase.h"
#include "minifi-cpp/core/ProcessorConfig.h"
#include "minifi-cpp/core/ProcessContext.h"
#include "minifi-cpp/core/ProcessorDescriptor.h"
@@ -376,6 +377,10 @@ class ProcessorDescriptorImpl : public ProcessorDescriptor {
impl_->setSupportedProperties(properties);
}
+ void setSupportedProperties(std::span<const Property> properties) override {
+ impl_->setSupportedProperties(properties);
+ }
+
private:
Processor* impl_;
};
@@ -497,7 +502,9 @@ void Processor::setLogBulletinLevel(logging::LOG_LEVEL level) {
}
void Processor::setLoggerCallback(const std::function<void(logging::LOG_LEVEL level, const std::string& message)>& callback) {
- impl_->setLoggerCallback(callback);
+ impl_->forEachLogger([&] (const std::shared_ptr<logging::Logger>& logger) {
+ std::dynamic_pointer_cast<logging::LoggerBase>(logger)->setLogCallback(callback);
+ });
}
std::chrono::steady_clock::time_point Processor::getYieldExpirationTime() const {
diff --git a/libminifi/src/core/extension/ExtensionManager.cpp b/libminifi/src/core/extension/ExtensionManager.cpp
index 0a661bf93..0d60e7c6b 100644
--- a/libminifi/src/core/extension/ExtensionManager.cpp
+++ b/libminifi/src/core/extension/ExtensionManager.cpp
@@ -57,11 +57,14 @@ ExtensionManager::ExtensionManager(const std::shared_ptr<Configure>& config): lo
if (!library) {
continue;
}
- if (!library->verify(logger_)) {
+ const auto library_type = library->verify(logger_);
+ if (library_type == internal::Invalid) {
logger_->log_warn("Skipping library '{}' at '{}': failed verification, different build?",
library->name, library->getFullPath());
continue;
}
+
+ logger_->log_trace("Verified library {} at {} as {} extension", library->name, library->getFullPath(), magic_enum::enum_name(library_type));
auto extension = std::make_unique<Extension>(library->name, library->getFullPath());
if (!extension->load()) {
// error already logged by method
diff --git a/libminifi/src/core/extension/Utils.cpp b/libminifi/src/core/extension/Utils.cpp
new file mode 100644
index 000000000..38e90ac15
--- /dev/null
+++ b/libminifi/src/core/extension/Utils.cpp
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "core/extension/Utils.h"
+#include "minifi-c/minifi-c.h"
+#include "minifi-cpp/agent/agent_version.h"
+#include "utils/RegexUtils.h"
+
+namespace org::apache::nifi::minifi::core::extension::internal {
+
+[[nodiscard]] bool LibraryDescriptor::verify_as_cpp_extension() const {
+ const auto path = getFullPath();
+ if (!std::filesystem::exists(path)) {
+ throw std::runtime_error{"File not found: " + path.string()};
+ }
+ const std::string_view begin_marker = "__EXTENSION_BUILD_IDENTIFIER_BEGIN__";
+ const std::string_view end_marker = "__EXTENSION_BUILD_IDENTIFIER_END__";
+ const std::string magic_constant = utils::string::join_pack(begin_marker, AgentBuild::BUILD_IDENTIFIER, end_marker);
+ return utils::file::contains(path, magic_constant);
+}
+
+[[nodiscard]] bool LibraryDescriptor::verify_as_c_extension(const std::shared_ptr<logging::Logger>& logger) const {
+ const auto path = getFullPath();
+ if (!std::filesystem::exists(path)) {
+ throw std::runtime_error{"File not found: " + path.string()};
+ }
+ const std::string_view api_tag_prefix = "MINIFI_API_VERSION=[";
+ if (auto version = utils::file::findSubstringWithPrefix(path, api_tag_prefix, api_tag_prefix.size() + 20)) {
+ utils::SVMatch match;
+ if (!utils::regexSearch(version.value(), match, utils::Regex{R"(MINIFI_API_VERSION=\[([0-9]+)\.([0-9]+)\.([0-9]+)\])"})) {
+ logger->log_error("Found api version in invalid format: '{}'", version.value());
+ return false;
+ }
+ gsl_Assert(match.size() == 4);
+ const int major = std::stoi(match[1]);
+ const int minor = std::stoi(match[2]);
+ const int patch = std::stoi(match[3]);
+ if (major != MINIFI_API_MAJOR_VERSION) {
+ logger->log_error("API major version mismatch, application is '{}' while extension is '{}.{}.{}'", MINIFI_API_VERSION, major, minor, patch);
+ return false;
+ }
+ if (minor > MINIFI_API_MINOR_VERSION) {
+ logger->log_error("Extension is built for a newer version, application is '{}' while extension is '{}.{}.{}'", MINIFI_API_VERSION, major, minor, patch);
+ return false;
+ }
+ return true;
+ }
+ return false;
+}
+
+} // namespace org::apache::nifi::minifi::core::extension::internal
diff --git a/libminifi/src/core/logging/LoggerBase.cpp b/libminifi/src/core/logging/LoggerBase.cpp
index e90149b43..d63365dfd 100644
--- a/libminifi/src/core/logging/LoggerBase.cpp
+++ b/libminifi/src/core/logging/LoggerBase.cpp
@@ -49,7 +49,18 @@ bool LoggerBase::should_log(LOG_LEVEL level) {
return delegate_->should_log(mapToSpdLogLevel(level));
}
+std::string LoggerBase::trimToMaxSizeAndAddId(std::string my_string) {
+ auto max_log_size = getMaxLogSize();
+ if (max_log_size >= 0 && my_string.size() > gsl::narrow<size_t>(max_log_size))
+ my_string = my_string.substr(0, max_log_size);
+ if (auto id = get_id()) {
+ my_string += *id;
+ }
+ return my_string;
+}
+
void LoggerBase::log_string(LOG_LEVEL level, std::string str) {
+ str = trimToMaxSizeAndAddId(std::move(str));
if (log_callback_) {
log_callback_(level, str);
}
diff --git a/libminifi/src/expression-language/Expression.cpp b/libminifi/src/expression-language/Expression.cpp
index f8de01751..e663a0dd3 100644
--- a/libminifi/src/expression-language/Expression.cpp
+++ b/libminifi/src/expression-language/Expression.cpp
@@ -35,6 +35,7 @@
#include "expression-language/Expression.h"
#include "utils/RegexUtils.h"
#include "utils/TimeUtil.h"
+#include "utils/TimeZoneUtils.h"
#ifdef WIN32
#pragma comment(lib, "wldap32.lib" )
@@ -398,7 +399,9 @@ Value expr_endsWith(const std::vector<Value> &args) {
}
Value expr_contains(const std::vector<Value> &args) {
- return Value(std::string::npos != args[0].asString().find(args[1].asString()));
+ const std::string &arg_0 = args[0].asString();
+ const std::string &arg_1 = args[1].asString();
+ return Value(arg_0.contains(arg_1));
}
Value expr_in(const std::vector<Value> &args) {
@@ -633,12 +636,13 @@ Value expr_escapeCsv(const std::vector<Value> &args) {
Value expr_format(const std::vector<Value> &args) {
using std::chrono::milliseconds;
- date::sys_time<milliseconds> utc_time_point{milliseconds(args[0].asUnsignedLong())};
- auto zone = args.size() > 2 ? date::locate_zone(args[2].asString()) : date::current_zone();
+ const date::sys_time<milliseconds> utc_time_point{milliseconds(args[0].asUnsignedLong())};
+ const auto format_string = args[1].asString();
+ const auto zone = args.size() > 2 ? date::locate_zone(args[2].asString()) : date::current_zone();
- auto zoned_time_point = date::make_zoned(zone, utc_time_point);
+ const auto zoned_time_point = date::make_zoned(zone, utc_time_point);
std::ostringstream result_stream;
- result_stream << date::format(args[1].asString(), zoned_time_point);
+ result_stream << date::format(format_string, zoned_time_point);
return Value(result_stream.str());
}
diff --git a/libminifi/src/minifi-c.cpp b/libminifi/src/minifi-c.cpp
index e00d54bb0..815998356 100644
--- a/libminifi/src/minifi-c.cpp
+++ b/libminifi/src/minifi-c.cpp
@@ -16,8 +16,28 @@
*/
#include "minifi-c/minifi-c.h"
-#include "properties/Configure.h"
-#include "core/extension/Extension.h"
+
+#include <memory>
+#include <vector>
+
+#include "agent/agent_docs.h"
+#include "core/ProcessorMetrics.h"
+#include "minifi-cpp/core/Annotation.h"
+#include "minifi-cpp/core/ClassLoader.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "minifi-cpp/core/ProcessSession.h"
+#include "minifi-cpp/core/ProcessorApi.h"
+#include "minifi-cpp/core/ProcessorDescriptor.h"
+#include "minifi-cpp/core/ProcessorFactory.h"
+#include "minifi-cpp/core/ProcessorMetadata.h"
+#include "minifi-cpp/core/PropertyValidator.h"
+#include "minifi-cpp/core/logging/Logger.h"
+#include "minifi-cpp/core/state/PublishedMetricProvider.h"
+#include "minifi-cpp/Exception.h"
+#include "core/extension/ExtensionManager.h"
+#include "utils/PropertyErrors.h"
+#include "minifi-cpp/agent/build_description.h"
+#include "utils/CProcessor.h"
namespace minifi = org::apache::nifi::minifi;
@@ -27,12 +47,202 @@ std::string toString(MinifiStringView sv) {
return {sv.data, sv.length};
}
+std::string_view toStringView(MinifiStringView sv) {
+ return {sv.data, sv.length};
+}
+
+minifi::core::annotation::Input toInputRequirement(MinifiInputRequirement req) {
+ switch (req) {
+ case MINIFI_INPUT_REQUIRED: return minifi::core::annotation::Input::INPUT_REQUIRED;
+ case MINIFI_INPUT_ALLOWED: return minifi::core::annotation::Input::INPUT_ALLOWED;
+ case MINIFI_INPUT_FORBIDDEN: return minifi::core::annotation::Input::INPUT_FORBIDDEN;
+ }
+ gsl_FailFast();
+}
+
+minifi::core::logging::LOG_LEVEL toLogLevel(MinifiLogLevel lvl) {
+ switch (lvl) {
+ case MINIFI_LOG_LEVEL_TRACE: return minifi::core::logging::trace;
+ case MINIFI_LOG_LEVEL_DEBUG: return minifi::core::logging::debug;
+ case MINIFI_LOG_LEVEL_INFO: return minifi::core::logging::info;
+ case MINIFI_LOG_LEVEL_WARNING: return minifi::core::logging::warn;
+ case MINIFI_LOG_LEVEL_ERROR: return minifi::core::logging::err;
+ case MINIFI_LOG_LEVEL_CRITICAL: return minifi::core::logging::critical;
+ case MINIFI_LOG_LEVEL_OFF: return minifi::core::logging::off;
+ }
+ gsl_FailFast();
+}
+
+gsl::not_null<const minifi::core::PropertyValidator*> toPropertyValidator(MinifiValidator validator) {
+ switch (validator) {
+ case MINIFI_VALIDATOR_ALWAYS_VALID: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::ALWAYS_VALID_VALIDATOR);
+ case MINIFI_VALIDATOR_NON_BLANK: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::NON_BLANK_VALIDATOR);
+ case MINIFI_VALIDATOR_TIME_PERIOD: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::TIME_PERIOD_VALIDATOR);
+ case MINIFI_VALIDATOR_BOOLEAN: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::BOOLEAN_VALIDATOR);
+ case MINIFI_VALIDATOR_INTEGER: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::INTEGER_VALIDATOR);
+ case MINIFI_VALIDATOR_UNSIGNED_INTEGER: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::UNSIGNED_INTEGER_VALIDATOR);
+ case MINIFI_VALIDATOR_DATA_SIZE: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::DATA_SIZE_VALIDATOR);
+ case MINIFI_VALIDATOR_PORT: return gsl::make_not_null(&minifi::core::StandardPropertyValidators::PORT_VALIDATOR);
+ }
+ gsl_FailFast();
+}
+
+minifi::core::Property createProperty(const MinifiPropertyDefinition* property_description) {
+ gsl_Expects(property_description);
+ std::vector<std::string_view> allowed_values;
+ allowed_values.reserve(property_description->allowed_values_count);
+ for (size_t i = 0; i < property_description->allowed_values_count; ++i) {
+ allowed_values.push_back(toStringView(property_description->allowed_values_ptr[i]));
+ }
+ std::vector<std::string_view> allowed_types;
+ if (property_description->type) {
+ allowed_types.push_back(toStringView(*property_description->type));
+ }
+ std::optional<std::string_view> default_value;
+ if (property_description->default_value) {
+ default_value = toStringView(*property_description->default_value);
+ }
+ return minifi::core::Property{minifi::core::PropertyReference{
+ toStringView(property_description->name),
+ toStringView(property_description->display_name),
+ toStringView(property_description->description),
+ property_description->is_required,
+ property_description->is_sensitive,
+ std::span(allowed_values),
+ std::span(allowed_types),
+ default_value,
+ toPropertyValidator(property_description->validator),
+ property_description->supports_expression_language
+ }};
+}
+
+class CProcessorFactory : public minifi::core::ProcessorFactory {
+ public:
+ CProcessorFactory(std::string group_name, std::string class_name, minifi::utils::CProcessorClassDescription class_description)
+ : group_name_(std::move(group_name)),
+ class_name_(std::move(class_name)),
+ class_description_(std::move(class_description)) {}
+ std::unique_ptr<minifi::core::ProcessorApi> create(minifi::core::ProcessorMetadata metadata) override {
+ return std::make_unique<minifi::utils::CProcessor>(class_description_, metadata);
+ }
+
+ [[nodiscard]] std::string getGroupName() const override {
+ return group_name_;
+ }
+
+ [[nodiscard]] std::string getClassName() const override {
+ return class_name_;
+ }
+
+ CProcessorFactory() = delete;
+ CProcessorFactory(const CProcessorFactory&) = delete;
+ CProcessorFactory& operator=(const CProcessorFactory&) = delete;
+ CProcessorFactory(CProcessorFactory&&) = delete;
+ CProcessorFactory& operator=(CProcessorFactory&&) = delete;
+ ~CProcessorFactory() override = default;
+
+ private:
+ std::string group_name_;
+ std::string class_name_;
+ minifi::utils::CProcessorClassDescription class_description_;
+};
+
} // namespace
+namespace org::apache::nifi::minifi::utils {
+
+void useCProcessorClassDescription(const MinifiProcessorClassDefinition& class_description, const std::function<void(minifi::ClassDescription, minifi::utils::CProcessorClassDescription)>& fn) {
+ std::vector<minifi::core::Property> properties;
+ properties.reserve(class_description.class_properties_count);
+ for (size_t i = 0; i < class_description.class_properties_count; ++i) {
+ properties.push_back(createProperty(&class_description.class_properties_ptr[i]));
+ }
+ std::vector<minifi::core::DynamicProperty> dynamic_properties;
+ dynamic_properties.reserve(class_description.dynamic_properties_count);
+ for (size_t i = 0; i < class_description.dynamic_properties_count; ++i) {
+ dynamic_properties.push_back(minifi::core::DynamicPropertyDefinition{
+ .name = toStringView(class_description.dynamic_properties_ptr[i].name),
+ .value = toStringView(class_description.dynamic_properties_ptr[i].value),
+ .description = toStringView(class_description.dynamic_properties_ptr[i].description),
+ .supports_expression_language = class_description.dynamic_properties_ptr[i].supports_expression_language
+ });
+ }
+ std::vector<minifi::core::Relationship> relationships;
+ relationships.reserve(class_description.class_relationships_count);
+ for (size_t i = 0; i < class_description.class_relationships_count; ++i) {
+ relationships.push_back(minifi::core::Relationship{
+ toString(class_description.class_relationships_ptr[i].name),
+ toString(class_description.class_relationships_ptr[i].description)
+ });
+ }
+ std::vector<minifi::core::OutputAttribute> output_attributes;
+ for (size_t attribute_idx = 0; attribute_idx < class_description.output_attributes_count; ++attribute_idx) {
+ minifi::core::OutputAttribute output_attribute{};
+ output_attribute.name = toString(class_description.output_attributes_ptr[attribute_idx].name);
+ output_attribute.description = toString(class_description.output_attributes_ptr[attribute_idx].description);
+ for (size_t rel_idx = 0; rel_idx < class_description.output_attributes_ptr[attribute_idx].relationships_count; ++rel_idx) {
+ auto output_attribute_rel_name = toStringView(class_description.output_attributes_ptr[attribute_idx].relationships_ptr[rel_idx]);
+ auto rel_it = std::find(relationships.begin(), relationships.end(), minifi::core::Relationship{std::string{output_attribute_rel_name}, ""});
+ gsl_Assert(rel_it != relationships.end());
+ output_attribute.relationships.push_back(*rel_it);
+ }
+ output_attributes.push_back(output_attribute);
+ }
+
+ auto name_segments = minifi::utils::string::split(toStringView(class_description.full_name), "::");
+ gsl_Assert(!name_segments.empty());
+
+ minifi::ClassDescription description{
+ .type_ = minifi::ResourceType::Processor,
+ .short_name_ = name_segments.back(),
+ .full_name_ = minifi::utils::string::join(".", name_segments),
+ .description_ = toString(class_description.description),
+ .class_properties_ = properties,
+ .dynamic_properties_ = dynamic_properties,
+ .class_relationships_ = relationships,
+ .output_attributes_ = output_attributes,
+ .supports_dynamic_properties_ = class_description.supports_dynamic_properties,
+ .supports_dynamic_relationships_ = class_description.supports_dynamic_relationships,
+ .inputRequirement_ = minifi::core::annotation::toString(toInputRequirement(class_description.input_requirement)),
+ .isSingleThreaded_ = class_description.is_single_threaded,
+ };
+
+ minifi::utils::CProcessorClassDescription c_class_description{
+ .name = name_segments.back(),
+ .class_properties = properties,
+ .class_relationships = relationships,
+ .supports_dynamic_properties = description.supports_dynamic_properties_,
+ .supports_dynamic_relationships = description.supports_dynamic_relationships_,
+ .input_requirement = toInputRequirement(class_description.input_requirement),
+ .is_single_threaded = description.isSingleThreaded_,
+
+ .callbacks = class_description.callbacks
+ };
+
+ fn(description, c_class_description);
+}
+
+} // namespace org::apache::nifi::minifi::utils
+
extern "C" {
-MinifiExtension* MinifiCreateExtension(const MinifiExtensionCreateInfo* extension_create_info) {
+MinifiExtension* MinifiCreateExtension(MinifiStringView /*api_version*/, const MinifiExtensionCreateInfo* extension_create_info) {
gsl_Assert(extension_create_info);
+ auto extension_name = toString(extension_create_info->name);
+ minifi::BundleDetails bundle{
+ .artifact = extension_name,
+ .group = "org.apache.nifi.minifi",
+ .version = toString(extension_create_info->version)
+ };
+ for (size_t proc_idx = 0; proc_idx < extension_create_info->processors_count; ++proc_idx) {
+ minifi::utils::useCProcessorClassDescription(extension_create_info->processors_ptr[proc_idx], [&] (const auto& description, const auto& c_class_description) {
+ minifi::ExternalBuildDescription::addExternalComponent(bundle, description);
+
+ minifi::core::ClassLoader::getDefaultClassLoader().getClassLoader(extension_name).registerClass(
+ c_class_description.name,
+ std::make_unique<CProcessorFactory>(extension_name, toString(extension_create_info->processors_ptr[proc_idx].full_name), c_class_description));
+ });
+ }
return reinterpret_cast<MinifiExtension*>(new org::apache::nifi::minifi::core::extension::Extension::Info{
.name = toString(extension_create_info->name),
.version = toString(extension_create_info->version),
@@ -41,14 +251,198 @@ MinifiExtension* MinifiCreateExtension(const MinifiExtensionCreateInfo* extensio
});
}
-void MinifiConfigureGet(MinifiConfig* config, MinifiStringView key, void(*cb)(void* user_data, MinifiStringView value), void* user_data) {
- gsl_Assert(config);
- auto value = reinterpret_cast<minifi::Configure*>(config)->get(toString(key));
+MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext* context, MinifiStringView property_name, MinifiFlowFile* flowfile,
+ void (*result_cb)(void* user_ctx, MinifiStringView result), void* user_ctx) {
+ gsl_Assert(context != MINIFI_NULL);
+ auto result = reinterpret_cast<minifi::core::ProcessContext*>(context)->getProperty(toStringView(property_name),
+ flowfile != MINIFI_NULL ? reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile)->get() : nullptr);
+ if (result) {
+ result_cb(user_ctx, MinifiStringView{.data = result.value().data(), .length = result.value().length()});
+ return MINIFI_STATUS_SUCCESS;
+ }
+ switch (static_cast<minifi::core::PropertyErrorCode>(result.error().value())) {
+ case minifi::core::PropertyErrorCode::NotSupportedProperty: return MINIFI_STATUS_NOT_SUPPORTED_PROPERTY;
+ case minifi::core::PropertyErrorCode::DynamicPropertiesNotSupported: return MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED;
+ case minifi::core::PropertyErrorCode::PropertyNotSet: return MINIFI_STATUS_PROPERTY_NOT_SET;
+ case minifi::core::PropertyErrorCode::ValidationFailed: return MINIFI_STATUS_VALIDATION_FAILED;
+ default: return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+}
+
+void MinifiProcessContextGetProcessorName(MinifiProcessContext* context, void(*cb)(void* user_ctx, MinifiStringView result), void* user_ctx) {
+ gsl_Assert(context != MINIFI_NULL);
+ auto name = reinterpret_cast<minifi::core::ProcessContext*>(context)->getProcessorInfo().getName();
+ cb(user_ctx, MinifiStringView{.data = name.data(), .length = name.length()});
+}
+
+MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext* context, MinifiStringView property_name) {
+ gsl_Assert(context != MINIFI_NULL);
+ return reinterpret_cast<minifi::core::ProcessContext*>(context)->hasNonEmptyProperty(toString(property_name));
+}
+
+void MinifiConfigGet(MinifiConfig* configure, MinifiStringView key, void(*cb)(void* user_ctx, MinifiStringView result), void* user_ctx) {
+ gsl_Assert(configure != MINIFI_NULL);
+ auto value = reinterpret_cast<minifi::Configure*>(configure)->get(toString(key));
if (value) {
- cb(user_data, MinifiStringView{
+ cb(user_ctx, MinifiStringView{
.data = value->data(),
- .length = gsl::narrow<size_t>(value->length())
+ .length = value->length()
+ });
+ }
+}
+
+void MinifiLoggerSetMaxLogSize(MinifiLogger* logger, int32_t max_size) {
+ gsl_Assert(logger != MINIFI_NULL);
+ (*reinterpret_cast<std::shared_ptr<minifi::core::logging::Logger>*>(logger))->set_max_log_size(max_size);
+}
+
+void MinifiLoggerLogString(MinifiLogger* logger, MinifiLogLevel level, MinifiStringView msg) {
+ gsl_Assert(logger != MINIFI_NULL);
+ (*reinterpret_cast<std::shared_ptr<minifi::core::logging::Logger>*>(logger))->log_string(toLogLevel(level), toString(msg));
+}
+
+MinifiBool MinifiLoggerShouldLog(MinifiLogger* logger, MinifiLogLevel level) {
+ gsl_Assert(logger != MINIFI_NULL);
+ return (*reinterpret_cast<std::shared_ptr<minifi::core::logging::Logger>*>(logger))->should_log(toLogLevel(level));
+}
+
+MinifiLogLevel MinifiLoggerLevel(MinifiLogger* logger) {
+ gsl_Assert(logger != MINIFI_NULL);
+ switch ((*reinterpret_cast<std::shared_ptr<minifi::core::logging::Logger>*>(logger))->level()) {
+ case minifi::core::logging::LOG_LEVEL::trace: return MINIFI_LOG_LEVEL_TRACE;
+ case minifi::core::logging::LOG_LEVEL::debug: return MINIFI_LOG_LEVEL_DEBUG;
+ case minifi::core::logging::LOG_LEVEL::info: return MINIFI_LOG_LEVEL_INFO;
+ case minifi::core::logging::LOG_LEVEL::warn: return MINIFI_LOG_LEVEL_WARNING;
+ case minifi::core::logging::LOG_LEVEL::err: return MINIFI_LOG_LEVEL_ERROR;
+ case minifi::core::logging::LOG_LEVEL::critical: return MINIFI_LOG_LEVEL_CRITICAL;
+ case minifi::core::logging::LOG_LEVEL::off: return MINIFI_LOG_LEVEL_OFF;
+ }
+ gsl_FailFast();
+}
+
+MINIFI_OWNED gsl::owner<MinifiPublishedMetrics*> MinifiPublishedMetricsCreate(const size_t count, const MinifiStringView* metric_names, const double* metric_values) {
+ const gsl::owner<std::vector<minifi::state::PublishedMetric>*> metrics = new std::vector<minifi::state::PublishedMetric>(); // NOLINT(modernize-use-auto)
+ metrics->reserve(count);
+ for (size_t i = 0; i < count; i++) {
+ metrics->emplace_back(minifi::state::PublishedMetric{toString(metric_names[i]), metric_values[i], {}});
+ }
+ return reinterpret_cast<MinifiPublishedMetrics*>(metrics);
+}
+
+MINIFI_OWNED MinifiFlowFile* MinifiProcessSessionGet(MinifiProcessSession* session) {
+ gsl_Assert(session != MINIFI_NULL);
+ if (const auto ff = reinterpret_cast<minifi::core::ProcessSession*>(session)->get()) {
+ return reinterpret_cast<MinifiFlowFile*>(new std::shared_ptr<minifi::core::FlowFile>(ff));
+ }
+ return MINIFI_NULL;
+}
+
+MINIFI_OWNED MinifiFlowFile* MinifiProcessSessionCreate(MinifiProcessSession* session, MinifiFlowFile* parent) {
+ gsl_Assert(session != MINIFI_NULL);
+ if (const auto ff = reinterpret_cast<minifi::core::ProcessSession*>(session)->create(parent != MINIFI_NULL
+ ? reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(parent)->get()
+ : nullptr)) {
+ return reinterpret_cast<MinifiFlowFile*>(new std::shared_ptr<minifi::core::FlowFile>(ff));
+ }
+ return MINIFI_NULL;
+}
+
+void MinifiProcessSessionTransfer(MinifiProcessSession* session, MINIFI_OWNED MinifiFlowFile* flowfile, MinifiStringView relationship_name) {
+ gsl_Assert(flowfile != MINIFI_NULL);
+ reinterpret_cast<minifi::core::ProcessSession*>(session)->transfer(
+ *reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile), minifi::core::Relationship{toString(relationship_name), ""});
+}
+
+void MinifiProcessSessionRemove(MinifiProcessSession* session, MINIFI_OWNED MinifiFlowFile* flowfile) {
+ gsl_Assert(flowfile != MINIFI_NULL);
+ reinterpret_cast<minifi::core::ProcessSession*>(session)->remove(*reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile));
+}
+
+MinifiStatus MinifiProcessSessionRead(MinifiProcessSession* session, MinifiFlowFile* flowfile, int64_t(*cb)(void* user_ctx, MinifiInputStream*), void* user_ctx) {
+ gsl_Assert(session != MINIFI_NULL);
+ gsl_Assert(flowfile != MINIFI_NULL);
+ try {
+ reinterpret_cast<minifi::core::ProcessSession*>(session)->read(*reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile), [&] (auto& input_stream) -> int64_t {
+ return cb(user_ctx, reinterpret_cast<MinifiInputStream*>(input_stream.get()));
});
+ return MINIFI_STATUS_SUCCESS;
+ } catch (...) {
+ return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+}
+
+MinifiStatus MinifiProcessSessionWrite(MinifiProcessSession* session, MinifiFlowFile* ff, int64_t(*cb)(void* user_ctx, MinifiOutputStream*), void* user_ctx) {
+ gsl_Assert(session != MINIFI_NULL);
+ gsl_Assert(ff != MINIFI_NULL);
+ try {
+ reinterpret_cast<minifi::core::ProcessSession*>(session)->write(*reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(ff), [&] (auto& output_stream) -> int64_t {
+ return cb(user_ctx, reinterpret_cast<MinifiOutputStream*>(output_stream.get()));
+ });
+ return MINIFI_STATUS_SUCCESS;
+ } catch (...) {
+ return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+}
+
+size_t MinifiInputStreamSize(MinifiInputStream* stream) {
+ gsl_Assert(stream != MINIFI_NULL);
+ return reinterpret_cast<minifi::io::InputStream*>(stream)->size();
+}
+
+int64_t MinifiInputStreamRead(MinifiInputStream* stream, char* buffer, size_t size) {
+ gsl_Assert(stream != MINIFI_NULL);
+ return gsl::narrow<int64_t>(reinterpret_cast<minifi::io::InputStream*>(stream)->read(std::span(reinterpret_cast<std::byte*>(buffer), size)));
+}
+
+int64_t MinifiOutputStreamWrite(MinifiOutputStream* stream, const char* data, size_t size) {
+ gsl_Assert(stream != MINIFI_NULL);
+ return gsl::narrow<int64_t>(reinterpret_cast<minifi::io::OutputStream*>(stream)->write(as_bytes(std::span(data, size))));
+}
+
+void MinifiStatusToString(MinifiStatus status, void(*cb)(void* user_ctx, MinifiStringView str), void* user_ctx) {
+ std::string message = [&] () -> std::string {
+ switch (status) {
+ case MINIFI_STATUS_SUCCESS: return "Success";
+ case MINIFI_STATUS_UNKNOWN_ERROR: return "Unknown error";
+ case MINIFI_STATUS_NOT_SUPPORTED_PROPERTY: return minifi::core::PropertyErrorCategory{}.message(static_cast<int>(minifi::core::PropertyErrorCode::NotSupportedProperty));
+ case MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED: return minifi::core::PropertyErrorCategory{}.message(static_cast<int>(minifi::core::PropertyErrorCode::DynamicPropertiesNotSupported));
+ case MINIFI_STATUS_PROPERTY_NOT_SET: return minifi::core::PropertyErrorCategory{}.message(static_cast<int>(minifi::core::PropertyErrorCode::PropertyNotSet));
+ case MINIFI_STATUS_VALIDATION_FAILED: return minifi::core::PropertyErrorCategory{}.message(static_cast<int>(minifi::core::PropertyErrorCode::ValidationFailed));
+ default: return "Unknown error";
+ }
+ }();
+ cb(user_ctx, MinifiStringView{.data = message.data(), .length = message.size()});
+}
+
+void MinifiFlowFileSetAttribute(MinifiProcessSession* session, MinifiFlowFile* flowfile, MinifiStringView attribute_name, const MinifiStringView* attribute_value) {
+ gsl_Assert(session != MINIFI_NULL);
+ gsl_Assert(flowfile != MINIFI_NULL);
+ if (attribute_value == nullptr) {
+ reinterpret_cast<minifi::core::ProcessSession*>(session)->removeAttribute(**reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile), toString(attribute_name));
+ } else {
+ reinterpret_cast<minifi::core::ProcessSession*>(session)->putAttribute(
+ **reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile), toString(attribute_name), toString(*attribute_value));
+ }
+}
+
+MinifiBool MinifiFlowFileGetAttribute(MinifiProcessSession* session, MinifiFlowFile* flowfile, MinifiStringView attribute_name,
+ void(*cb)(void* user_ctx, MinifiStringView attribute_value), void* user_ctx) {
+ gsl_Assert(session != MINIFI_NULL);
+ gsl_Assert(flowfile != MINIFI_NULL);
+ auto value = (*reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile))->getAttribute(toString(attribute_name));
+ if (!value.has_value()) {
+ return false;
+ }
+ cb(user_ctx, MinifiStringView{.data = value->data(), .length = value->size()});
+ return true;
+}
+
+void MinifiFlowFileGetAttributes(MinifiProcessSession* session, MinifiFlowFile* flowfile,
+ void(*cb)(void* user_ctx, MinifiStringView attribute_name, MinifiStringView attribute_value), void* user_ctx) {
+ gsl_Assert(session != MINIFI_NULL);
+ gsl_Assert(flowfile != MINIFI_NULL);
+ for (auto& [key, value] : (*reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile))->getAttributes()) {
+ cb(user_ctx, MinifiStringView{.data = key.data(), .length = key.size()}, MinifiStringView{.data = value.data(), .length = value.size()});
}
}
diff --git a/libminifi/src/utils/CProcessor.cpp b/libminifi/src/utils/CProcessor.cpp
new file mode 100644
index 000000000..6285cb22e
--- /dev/null
+++ b/libminifi/src/utils/CProcessor.cpp
@@ -0,0 +1,45 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "utils/CProcessor.h"
+
+namespace org::apache::nifi::minifi::utils {
+
+std::vector<minifi::state::PublishedMetric> CProcessor::getCustomMetrics() const {
+ if (class_description_.callbacks.calculateMetrics == nullptr) {
+ return {};
+ }
+ std::unique_ptr<std::vector<minifi::state::PublishedMetric>> metrics{reinterpret_cast<std::vector<minifi::state::PublishedMetric>*>(class_description_.callbacks.calculateMetrics(impl_))};
+ return *metrics;
+}
+
+std::vector<minifi::state::response::SerializedResponseNode> CProcessorMetricsWrapper::serialize() {
+ std::vector<minifi::state::response::SerializedResponseNode> nodes;
+ for (auto& custom_node : source_processor_.getCustomMetrics()) {
+ nodes.push_back(minifi::state::response::SerializedResponseNode{
+ .name = utils::string::snakeCaseToPascalCase(custom_node.name),
+ .value = static_cast<uint64_t>(custom_node.value)
+ });
+ }
+ return nodes;
+}
+
+std::vector<minifi::state::PublishedMetric> CProcessorMetricsWrapper::calculateMetrics() {
+ return source_processor_.getCustomMetrics();
+}
+
+} // namespace org::apache::nifi::minifi::utils
diff --git a/libminifi/src/utils/TimeUtil.cpp b/libminifi/src/utils/TimeUtil.cpp
index 66fd07d8f..a378e7edc 100644
--- a/libminifi/src/utils/TimeUtil.cpp
+++ b/libminifi/src/utils/TimeUtil.cpp
@@ -16,6 +16,7 @@
*/
#include "utils/TimeUtil.h"
+#include "minifi-cpp/utils/TimeUtil.h"
#ifdef WIN32
#include "date/tz.h"
diff --git a/libminifi/test/libtest/integration/ConnectionCountingServer.cpp b/libminifi/test/libtest/integration/ConnectionCountingServer.cpp
index 6a1fc4409..34ef54cc3 100644
--- a/libminifi/test/libtest/integration/ConnectionCountingServer.cpp
+++ b/libminifi/test/libtest/integration/ConnectionCountingServer.cpp
@@ -18,6 +18,7 @@
#include "ConnectionCountingServer.h"
#include <cinttypes>
+#include <cassert>
#include "utils/Id.h"
diff --git a/libminifi/test/unit/CronTests.cpp b/libminifi/test/unit/CronTests.cpp
index 1f3800577..82c73e95e 100644
--- a/libminifi/test/unit/CronTests.cpp
+++ b/libminifi/test/unit/CronTests.cpp
@@ -21,13 +21,11 @@
#include "date/date.h"
#include "date/tz.h"
#include "utils/TimeUtil.h"
+#include "utils/TimeZoneUtils.h"
using std::chrono::system_clock;
using std::chrono::seconds;
using org::apache::nifi::minifi::utils::Cron;
-#ifdef WIN32
-namespace timeutils = org::apache::nifi::minifi::utils::timeutils;
-#endif
void checkNext(const std::string& expr, const date::zoned_time<seconds>& from, const date::zoned_time<seconds>& next) {
@@ -164,7 +162,7 @@ TEST_CASE("Cron::calculateNextTrigger", "[cron]") {
using namespace date::literals; // NOLINT(google-build-using-namespace)
using namespace std::literals::chrono_literals;
#ifdef WIN32
- timeutils::dateSetInstall(TZ_DATA_DIR);
+ org::apache::nifi::minifi::utils::timeutils::dateSetInstall(TZ_DATA_DIR);
#endif
checkNext("0/15 * 1-4 * * ?",
@@ -450,7 +448,7 @@ TEST_CASE("Cron::calculateNextTrigger with timezones", "[cron]") {
using namespace date::literals; // NOLINT(google-build-using-namespace)
using namespace std::literals::chrono_literals;
#ifdef WIN32
- timeutils::dateSetInstall(TZ_DATA_DIR);
+ org::apache::nifi::minifi::utils::timeutils::dateSetInstall(TZ_DATA_DIR);
#endif
const std::vector<std::string> time_zones{ "Europe/Berlin", "Asia/Seoul", "America/Los_Angeles", "Asia/Singapore", "UTC" };
diff --git a/libminifi/test/unit/ExtensionVerificationTests.cpp b/libminifi/test/unit/ExtensionVerificationTests.cpp
index 5139e18a5..71a425ada 100644
--- a/libminifi/test/unit/ExtensionVerificationTests.cpp
+++ b/libminifi/test/unit/ExtensionVerificationTests.cpp
@@ -22,6 +22,7 @@
#include "minifi-cpp/agent/agent_version.h"
#include "core/extension/Utils.h"
#include "unit/TestUtils.h"
+#include "minifi-c/minifi-c.h"
using namespace std::literals;
@@ -53,7 +54,15 @@ TEST_CASE_METHOD(Fixture, "Could load extension with matching build id") {
auto lib = minifi::core::extension::internal::asDynamicLibrary(extension_);
REQUIRE(lib);
- REQUIRE(lib->verify(logger));
+ CHECK(lib->verify(logger) == core::extension::internal::Cpp);
+}
+
+TEST_CASE_METHOD(Fixture, "Could load extension with matching C api") {
+ std::ofstream{extension_} << MINIFI_API_VERSION_TAG;
+
+ auto lib = minifi::core::extension::internal::asDynamicLibrary(extension_);
+ REQUIRE(lib);
+ CHECK(lib->verify(logger) == core::extension::internal::CApi);
}
TEST_CASE_METHOD(Fixture, "Can't load extension if the build id begin marker is missing") {
@@ -62,7 +71,7 @@ TEST_CASE_METHOD(Fixture, "Can't load extension if the build id begin marker is
auto lib = minifi::core::extension::internal::asDynamicLibrary(extension_);
REQUIRE(lib);
- REQUIRE_FALSE(lib->verify(logger));
+ CHECK(lib->verify(logger) == core::extension::internal::Invalid);
}
TEST_CASE_METHOD(Fixture, "Can't load extension if the build id end marker is missing") {
@@ -71,7 +80,7 @@ TEST_CASE_METHOD(Fixture, "Can't load extension if the build id end marker is mi
auto lib = minifi::core::extension::internal::asDynamicLibrary(extension_);
REQUIRE(lib);
- REQUIRE_FALSE(lib->verify(logger));
+ CHECK(lib->verify(logger) == core::extension::internal::Invalid);
}
TEST_CASE_METHOD(Fixture, "Can't load extension if the build id does not match") {
@@ -80,7 +89,7 @@ TEST_CASE_METHOD(Fixture, "Can't load extension if the build id does not match")
auto lib = minifi::core::extension::internal::asDynamicLibrary(extension_);
REQUIRE(lib);
- REQUIRE_FALSE(lib->verify(logger));
+ CHECK(lib->verify(logger) == core::extension::internal::Invalid);
}
TEST_CASE_METHOD(Fixture, "Can't load extension if the file does not exist") {
@@ -94,5 +103,5 @@ TEST_CASE_METHOD(Fixture, "Can't load extension if the file has zero length") {
auto lib = minifi::core::extension::internal::asDynamicLibrary(extension_);
REQUIRE(lib);
- REQUIRE_FALSE(lib->verify(logger));
+ CHECK(lib->verify(logger) == core::extension::internal::Invalid);
}
diff --git a/libminifi/test/unit/FileUtilsTests.cpp b/libminifi/test/unit/FileUtilsTests.cpp
index 71733473d..d8c2f93ac 100644
--- a/libminifi/test/unit/FileUtilsTests.cpp
+++ b/libminifi/test/unit/FileUtilsTests.cpp
@@ -553,3 +553,14 @@ TEST_CASE("file_clock to system_clock conversion tests") {
CHECK(file_time_t1-file_time_from_t0 < 10ms);
}
}
+
+TEST_CASE("FileUtils::findSubstringWithPrefix", "[TestFindSubstringWithPrefix]") {
+ std::stringstream ss;
+ ss << "garbage tag=1.2.3garbage tag=5.6.7";
+ SECTION("Exists") {
+ CHECK(utils::file::findSubstringWithPrefix(ss, "tag=", 9) == "tag=1.2.3");
+ }
+ SECTION("Does not exist") {
+ CHECK(utils::file::findSubstringWithPrefix(ss, "notag=", 9) == std::nullopt);
+ }
+}
diff --git a/libminifi/test/unit/SchedulingAgentTests.cpp b/libminifi/test/unit/SchedulingAgentTests.cpp
index ed79f7f18..e23b99e06 100644
--- a/libminifi/test/unit/SchedulingAgentTests.cpp
+++ b/libminifi/test/unit/SchedulingAgentTests.cpp
@@ -17,13 +17,15 @@
#include <chrono>
+#include "ProcessSessionFactory.h"
#include "unit/Catch.h"
-#include "unit/TestBase.h"
#include "unit/ProvenanceTestHelper.h"
+#include "unit/TestBase.h"
#include "unit/TestUtils.h"
#include "utils/TimeUtil.h"
#include "core/ProcessContextImpl.h"
#include "core/ProcessSessionFactory.h"
+#include "utils/TimeZoneUtils.h"
using namespace std::literals::chrono_literals;
diff --git a/libminifi/test/unit/StringUtilsTests.cpp b/libminifi/test/unit/StringUtilsTests.cpp
index b074e9381..3c05068e6 100644
--- a/libminifi/test/unit/StringUtilsTests.cpp
+++ b/libminifi/test/unit/StringUtilsTests.cpp
@@ -630,6 +630,16 @@ TEST_CASE("string::replaceEscapedCharacters tests") {
CHECK(string::replaceEscapedCharacters(R"(\\n)") == "\\n");
}
+TEST_CASE("string::snakeCaseToPascalCase tests") {
+ CHECK(string::snakeCaseToPascalCase("NotAThing") == "Notathing");
+ CHECK(string::snakeCaseToPascalCase("one_thing_other") == "OneThingOther");
+ CHECK(string::snakeCaseToPascalCase("trailing_underSCORE_") == "TrailingUnderscore");
+ CHECK(string::snakeCaseToPascalCase("HTTP_GET_count") == "HttpGetCount");
+ CHECK(string::snakeCaseToPascalCase("_leading_underscore") == "LeadingUnderscore");
+ CHECK(string::snakeCaseToPascalCase("multiple__underscores") == "MultipleUnderscores");
+ CHECK(string::snakeCaseToPascalCase("some_123_\n\rchars") == "Some123\n\rchars");
+}
+
#ifdef WIN32
TEST_CASE("Conversion from UTF-8 strings to UTF-16 strings works") {
using org::apache::nifi::minifi::utils::to_wstring;
diff --git a/minifi-api/CMakeLists.txt b/minifi-api/CMakeLists.txt
index 26949c2f9..aa684c154 100644
--- a/minifi-api/CMakeLists.txt
+++ b/minifi-api/CMakeLists.txt
@@ -1,3 +1,24 @@
+add_library(minifi-api-common INTERFACE)
+target_include_directories(minifi-api-common INTERFACE common/include)
+target_link_libraries(minifi-api-common INTERFACE gsl-lite)
+
add_library(minifi-api INTERFACE)
target_include_directories(minifi-api INTERFACE include)
-target_link_libraries(minifi-api INTERFACE gsl-lite)
+target_link_libraries(minifi-api INTERFACE minifi-api-common)
+
+add_library(minifi-c-api INTERFACE)
+target_include_directories(minifi-c-api INTERFACE include/minifi-c)
+if (APPLE)
+ target_link_options(minifi-c-api INTERFACE -undefined dynamic_lookup)
+endif()
+
+if (WIN32)
+ set(IMPORT_LIB_FILE ${CMAKE_CURRENT_BINARY_DIR}/minifi-c-api.lib)
+ add_custom_target(minifi_c_api_import_lib
+ COMMAND lib.exe /def:${CMAKE_CURRENT_SOURCE_DIR}/minifi-c-api.def /out:${IMPORT_LIB_FILE} /machine:x64
+ BYPRODUCTS ${IMPORT_LIB_FILE}
+ )
+
+ add_dependencies(minifi-c-api minifi_c_api_import_lib)
+ target_link_libraries(minifi-c-api INTERFACE ${IMPORT_LIB_FILE})
+endif()
diff --git a/minifi-api/include/minifi-cpp/Exception.h b/minifi-api/common/include/minifi-cpp/Exception.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/Exception.h
rename to minifi-api/common/include/minifi-cpp/Exception.h
diff --git a/minifi-api/include/minifi-cpp/core/Annotation.h b/minifi-api/common/include/minifi-cpp/core/Annotation.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/core/Annotation.h
rename to minifi-api/common/include/minifi-cpp/core/Annotation.h
diff --git a/minifi-api/include/minifi-cpp/core/DynamicProperty.h b/minifi-api/common/include/minifi-cpp/core/DynamicPropertyDefinition.h
similarity index 90%
copy from minifi-api/include/minifi-cpp/core/DynamicProperty.h
copy to minifi-api/common/include/minifi-cpp/core/DynamicPropertyDefinition.h
index 263326acb..4fa36dcf4 100644
--- a/minifi-api/include/minifi-cpp/core/DynamicProperty.h
+++ b/minifi-api/common/include/minifi-cpp/core/DynamicPropertyDefinition.h
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -20,7 +20,7 @@
namespace org::apache::nifi::minifi::core {
-struct DynamicProperty {
+struct DynamicPropertyDefinition {
std::string_view name;
std::string_view value;
std::string_view description;
diff --git a/minifi-api/include/minifi-cpp/core/OutputAttributeDefinition.h b/minifi-api/common/include/minifi-cpp/core/OutputAttributeDefinition.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/core/OutputAttributeDefinition.h
rename to minifi-api/common/include/minifi-cpp/core/OutputAttributeDefinition.h
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/minifi-api/common/include/minifi-cpp/core/ProcessorMetadata.h
similarity index 100%
copy from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
copy to minifi-api/common/include/minifi-cpp/core/ProcessorMetadata.h
diff --git a/minifi-api/include/minifi-cpp/core/PropertyDefinition.h b/minifi-api/common/include/minifi-cpp/core/PropertyDefinition.h
similarity index 98%
rename from minifi-api/include/minifi-cpp/core/PropertyDefinition.h
rename to minifi-api/common/include/minifi-cpp/core/PropertyDefinition.h
index 182cd2e56..ad538e00b 100644
--- a/minifi-api/include/minifi-cpp/core/PropertyDefinition.h
+++ b/minifi-api/common/include/minifi-cpp/core/PropertyDefinition.h
@@ -22,7 +22,7 @@
#include <string_view>
#include <utility>
-#include "PropertyValidator.h"
+#include "minifi-cpp/core/PropertyValidator.h"
#include "minifi-cpp/utils/gsl.h"
namespace org::apache::nifi::minifi::core {
diff --git a/minifi-api/include/minifi-cpp/core/PropertyValidator.h b/minifi-api/common/include/minifi-cpp/core/PropertyValidator.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/core/PropertyValidator.h
rename to minifi-api/common/include/minifi-cpp/core/PropertyValidator.h
diff --git a/minifi-api/include/minifi-cpp/core/Relationship.h b/minifi-api/common/include/minifi-cpp/core/Relationship.h
similarity index 94%
rename from minifi-api/include/minifi-cpp/core/Relationship.h
rename to minifi-api/common/include/minifi-cpp/core/Relationship.h
index 15880bed4..f80f1ab76 100644
--- a/minifi-api/include/minifi-cpp/core/Relationship.h
+++ b/minifi-api/common/include/minifi-cpp/core/Relationship.h
@@ -38,6 +38,10 @@ class Relationship {
: Relationship{std::string{relationship_definition.name}, std::string{relationship_definition.description}} {
}
+ [[nodiscard]] RelationshipDefinition getDefinition() const {
+ return RelationshipDefinition{name_, description_};
+ }
+
[[nodiscard]] std::string getName() const {
return name_;
}
diff --git a/minifi-api/include/minifi-cpp/core/RelationshipDefinition.h b/minifi-api/common/include/minifi-cpp/core/RelationshipDefinition.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/core/RelationshipDefinition.h
rename to minifi-api/common/include/minifi-cpp/core/RelationshipDefinition.h
diff --git a/minifi-api/common/include/minifi-cpp/core/SpecialFlowAttribute.h b/minifi-api/common/include/minifi-cpp/core/SpecialFlowAttribute.h
new file mode 100644
index 000000000..225305a5b
--- /dev/null
+++ b/minifi-api/common/include/minifi-cpp/core/SpecialFlowAttribute.h
@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+namespace org::apache::nifi::minifi::core {
+
+struct SpecialFlowAttribute {
+ static constexpr std::string_view PATH = "path";
+ static constexpr std::string_view ABSOLUTE_PATH = "absolute.path";
+ static constexpr std::string_view FILENAME = "filename";
+ static constexpr std::string_view UUID = "uuid";
+ static constexpr std::string_view priority = "priority";
+ static constexpr std::string_view MIME_TYPE = "mime.type";
+ static constexpr std::string_view DISCARD_REASON = "discard.reason";
+ static constexpr std::string_view ALTERNATE_IDENTIFIER = "alternate.identifier";
+ static constexpr std::string_view FLOW_ID = "flow.id";
+
+ static constexpr std::array<std::string_view, 9> getSpecialFlowAttributes() {
+ return {PATH, ABSOLUTE_PATH, FILENAME, UUID, priority, MIME_TYPE, DISCARD_REASON, ALTERNATE_IDENTIFIER, FLOW_ID};
+ }
+};
+
+} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/logging/Logger.h b/minifi-api/common/include/minifi-cpp/core/logging/Logger.h
similarity index 74%
rename from minifi-api/include/minifi-cpp/core/logging/Logger.h
rename to minifi-api/common/include/minifi-cpp/core/logging/Logger.h
index 0eac6077f..e09aa5fbd 100644
--- a/minifi-api/include/minifi-cpp/core/logging/Logger.h
+++ b/minifi-api/common/include/minifi-cpp/core/logging/Logger.h
@@ -19,20 +19,13 @@
#include <string>
#include <mutex>
-#include <memory>
-#include <optional>
-#include <sstream>
#include <utility>
-#include <iostream>
#include <vector>
#include <algorithm>
#include "minifi-cpp/utils/gsl.h"
-#include "utils/Enum.h"
#include "utils/GeneralUtils.h"
-#include "fmt/chrono.h"
#include "fmt/std.h"
-#include "fmt/ostream.h"
namespace org::apache::nifi::minifi::core::logging {
@@ -92,41 +85,19 @@ class Logger {
}
virtual void set_max_log_size(int size) = 0;
- virtual std::optional<std::string> get_id() = 0;
virtual void log_string(LOG_LEVEL level, std::string str) = 0;
virtual bool should_log(LOG_LEVEL level) = 0;
[[nodiscard]] virtual LOG_LEVEL level() const = 0;
virtual ~Logger() = default;
- virtual void setLogCallback(const std::function<void(LOG_LEVEL level, const std::string&)>& callback) = 0;
-
- protected:
- virtual int getMaxLogSize() = 0;
-
private:
- std::string trimToMaxSizeAndAddId(std::string my_string) {
- auto max_log_size = getMaxLogSize();
- if (max_log_size >= 0 && my_string.size() > gsl::narrow<size_t>(max_log_size))
- my_string = my_string.substr(0, max_log_size);
- if (auto id = get_id()) {
- my_string += *id;
- }
- return my_string;
- }
-
- template<typename ...Args>
- std::string stringify(fmt::format_string<Args...> fmt, Args&&... args) {
- auto log_message = fmt::format(std::move(fmt), std::forward<Args>(args)...);
- return trimToMaxSizeAndAddId(std::move(log_message));
- }
-
template<typename ...Args>
- inline void log(LOG_LEVEL level, log_format_string<Args...> fmt, Args&& ...args) {
+ void log(LOG_LEVEL level, log_format_string<Args...> fmt, Args&& ...args) {
if (!should_log(level)) {
return;
}
- log_string(level, stringify(std::move(fmt), map_args(std::forward<Args>(args))...));
+ log_string(level, fmt::format(std::move(fmt), map_args(std::forward<Args>(args))...));
}
};
diff --git a/minifi-api/include/minifi-cpp/io/InputStream.h b/minifi-api/common/include/minifi-cpp/io/InputStream.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/io/InputStream.h
rename to minifi-api/common/include/minifi-cpp/io/InputStream.h
diff --git a/minifi-api/include/minifi-cpp/io/OutputStream.h b/minifi-api/common/include/minifi-cpp/io/OutputStream.h
similarity index 99%
rename from minifi-api/include/minifi-cpp/io/OutputStream.h
rename to minifi-api/common/include/minifi-cpp/io/OutputStream.h
index d0e7f3d55..bae96376c 100644
--- a/minifi-api/include/minifi-cpp/io/OutputStream.h
+++ b/minifi-api/common/include/minifi-cpp/io/OutputStream.h
@@ -24,7 +24,7 @@
#include "Stream.h"
#include "minifi-cpp/utils/gsl.h"
#include "utils/SmallString.h"
-#include "utils/Id.h"
+#include "minifi-cpp/utils/Id.h"
namespace org::apache::nifi::minifi::io {
diff --git a/minifi-api/include/minifi-cpp/io/Stream.h b/minifi-api/common/include/minifi-cpp/io/Stream.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/io/Stream.h
rename to minifi-api/common/include/minifi-cpp/io/Stream.h
diff --git a/minifi-api/include/minifi-cpp/io/StreamCallback.h b/minifi-api/common/include/minifi-cpp/io/StreamCallback.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/io/StreamCallback.h
rename to minifi-api/common/include/minifi-cpp/io/StreamCallback.h
diff --git a/minifi-api/include/minifi-cpp/utils/Id.h b/minifi-api/common/include/minifi-cpp/utils/Id.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/utils/Id.h
rename to minifi-api/common/include/minifi-cpp/utils/Id.h
diff --git a/minifi-api/include/minifi-cpp/utils/Literals.h b/minifi-api/common/include/minifi-cpp/utils/Literals.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/utils/Literals.h
rename to minifi-api/common/include/minifi-cpp/utils/Literals.h
diff --git a/minifi-api/include/minifi-cpp/utils/SmallString.h b/minifi-api/common/include/minifi-cpp/utils/SmallString.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/utils/SmallString.h
rename to minifi-api/common/include/minifi-cpp/utils/SmallString.h
diff --git a/minifi-api/include/minifi-cpp/utils/gsl.h b/minifi-api/common/include/minifi-cpp/utils/gsl.h
similarity index 100%
rename from minifi-api/include/minifi-cpp/utils/gsl.h
rename to minifi-api/common/include/minifi-cpp/utils/gsl.h
diff --git a/minifi-api/include/minifi-c/minifi-c.h b/minifi-api/include/minifi-c/minifi-c.h
index 4c7585513..1f22c908c 100644
--- a/minifi-api/include/minifi-c/minifi-c.h
+++ b/minifi-api/include/minifi-c/minifi-c.h
@@ -24,24 +24,202 @@ extern "C" {
#include <stddef.h>
#include <stdint.h>
+#if __STDC_VERSION__ < 202311l
+# include <stdbool.h>
+#endif // < C23
-typedef struct MinifiConfig MinifiConfig;
-typedef struct MinifiExtension MinifiExtension;
+#define MINIFI_PRIVATE_STRINGIFY_HELPER(X) #X
+#define MINIFI_PRIVATE_STRINGIFY(X) MINIFI_PRIVATE_STRINGIFY_HELPER(X)
+
+#define MINIFI_API_MAJOR_VERSION 0
+#define MINIFI_API_MINOR_VERSION 1
+#define MINIFI_API_PATCH_VERSION 0
+#define MINIFI_API_VERSION MINIFI_PRIVATE_STRINGIFY(MINIFI_API_MAJOR_VERSION) "." MINIFI_PRIVATE_STRINGIFY(MINIFI_API_MINOR_VERSION) "." MINIFI_PRIVATE_STRINGIFY(MINIFI_API_PATCH_VERSION)
+#define MINIFI_API_VERSION_TAG "MINIFI_API_VERSION=[" MINIFI_API_VERSION "]"
+#define MINIFI_NULL nullptr
+#define MINIFI_OWNED
+
+typedef bool MinifiBool;
+
+typedef enum MinifiInputRequirement {
+ MINIFI_INPUT_REQUIRED = 0,
+ MINIFI_INPUT_ALLOWED = 1,
+ MINIFI_INPUT_FORBIDDEN = 2
+} MinifiInputRequirement;
+/// Represents a non-owning read-only view to a sized, not necessarily null-terminated string.
+/// invariant: [data, data + length) is a valid range
typedef struct MinifiStringView {
+ /// nullable, non-owning pointer to the beginning of a continuous character sequence
const char* data;
+ /// the length of the buffer pointed-to by data
size_t length;
} MinifiStringView;
+/// Represents an output relationship of a processor, part of the processor metadata
+typedef struct MinifiRelationshipDefinition {
+ /// Name, processors use this to transfer flow files to the connections connected to this relationship using MinifiProcessSessionTransfer.
+ MinifiStringView name;
+ /// Human-readable description of what flow files get routed to this relationship. Included in C2 manifest and generated processor docs.
+ MinifiStringView description;
+} MinifiRelationshipDefinition;
+
+typedef struct MinifiOutputAttributeDefinition {
+ MinifiStringView name;
+ size_t relationships_count;
+ const MinifiStringView* relationships_ptr;
+ MinifiStringView description;
+} MinifiOutputAttributeDefinition;
+
+typedef struct MinifiDynamicPropertyDefinition {
+ MinifiStringView name;
+ MinifiStringView value;
+ MinifiStringView description;
+ MinifiBool supports_expression_language;
+} MinifiDynamicPropertyDefinition;
+
+typedef struct MinifiFlowFile MinifiFlowFile;
+typedef struct MinifiLogger MinifiLogger;
+typedef struct MinifiProcessContext MinifiProcessContext;
+typedef struct MinifiProcessSession MinifiProcessSession;
+typedef struct MinifiInputStream MinifiInputStream;
+typedef struct MinifiOutputStream MinifiOutputStream;
+typedef struct MinifiConfig MinifiConfig;
+typedef struct MinifiExtension MinifiExtension;
+typedef struct MinifiPublishedMetrics MinifiPublishedMetrics;
+
+typedef enum MinifiStatus {
+ MINIFI_STATUS_SUCCESS = 0,
+ MINIFI_STATUS_UNKNOWN_ERROR = 1,
+ MINIFI_STATUS_NOT_SUPPORTED_PROPERTY = 2,
+ MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED = 3,
+ MINIFI_STATUS_PROPERTY_NOT_SET = 4,
+ MINIFI_STATUS_VALIDATION_FAILED = 5,
+ MINIFI_STATUS_PROCESSOR_YIELD = 6
+} MinifiStatus;
+
+typedef enum MinifiValidator {
+ MINIFI_VALIDATOR_ALWAYS_VALID = 0,
+ MINIFI_VALIDATOR_NON_BLANK = 1,
+ MINIFI_VALIDATOR_TIME_PERIOD = 2,
+ MINIFI_VALIDATOR_BOOLEAN = 3,
+ MINIFI_VALIDATOR_INTEGER = 4,
+ MINIFI_VALIDATOR_UNSIGNED_INTEGER = 5,
+ MINIFI_VALIDATOR_DATA_SIZE = 6,
+ MINIFI_VALIDATOR_PORT = 7
+} MinifiValidator;
+
+typedef struct MinifiPropertyDefinition {
+ MinifiStringView name;
+ MinifiStringView display_name;
+ MinifiStringView description;
+ MinifiBool is_required;
+ MinifiBool is_sensitive;
+
+ const MinifiStringView* default_value;
+ size_t allowed_values_count;
+ const MinifiStringView* allowed_values_ptr;
+ MinifiValidator validator;
+
+ const MinifiStringView* type;
+ MinifiBool supports_expression_language;
+} MinifiPropertyDefinition;
+
+typedef enum MinifiLogLevel {
+ MINIFI_LOG_LEVEL_TRACE = 0,
+ MINIFI_LOG_LEVEL_DEBUG = 1,
+ MINIFI_LOG_LEVEL_INFO = 2,
+ MINIFI_LOG_LEVEL_WARNING = 3,
+ MINIFI_LOG_LEVEL_ERROR = 4,
+ MINIFI_LOG_LEVEL_CRITICAL = 5,
+ MINIFI_LOG_LEVEL_OFF = 6
+} MinifiLogLevel;
+
+typedef struct MinifiProcessorMetadata {
+ MinifiStringView uuid;
+ MinifiStringView name;
+ MinifiLogger* logger; // borrowed reference, live until the processor is live
+} MinifiProcessorMetadata;
+
+typedef struct MinifiProcessorCallbacks {
+ MINIFI_OWNED void*(*create)(MinifiProcessorMetadata);
+ void(*destroy)(MINIFI_OWNED void*);
+ MinifiBool(*isWorkAvailable)(void*);
+ void(*restore)(void*, MINIFI_OWNED MinifiFlowFile*);
+ MinifiBool(*getTriggerWhenEmpty)(void*);
+ MinifiStatus(*onTrigger)(void*, MinifiProcessContext*, MinifiProcessSession*);
+ MinifiStatus(*onSchedule)(void*, MinifiProcessContext*);
+ void(*onUnSchedule)(void*);
+ MINIFI_OWNED MinifiPublishedMetrics*(*calculateMetrics)(void*);
+} MinifiProcessorCallbacks;
+
+typedef struct MinifiProcessorClassDefinition {
+ MinifiStringView full_name; // '::'-delimited fully qualified name e.g. 'org::apache::nifi::minifi::GenerateFlowFile'
+ MinifiStringView description;
+ size_t class_properties_count;
+ const MinifiPropertyDefinition* class_properties_ptr;
+ size_t dynamic_properties_count;
+ const MinifiDynamicPropertyDefinition* dynamic_properties_ptr;
+ size_t class_relationships_count;
+ const MinifiRelationshipDefinition* class_relationships_ptr;
+ size_t output_attributes_count;
+ const MinifiOutputAttributeDefinition* output_attributes_ptr;
+ MinifiBool supports_dynamic_properties;
+ MinifiBool supports_dynamic_relationships;
+ MinifiInputRequirement input_requirement;
+ MinifiBool is_single_threaded;
+
+ MinifiProcessorCallbacks callbacks;
+} MinifiProcessorClassDefinition;
+
typedef struct MinifiExtensionCreateInfo {
MinifiStringView name;
MinifiStringView version;
void(*deinit)(void* user_data);
void* user_data;
+ size_t processors_count;
+ const MinifiProcessorClassDefinition* processors_ptr;
} MinifiExtensionCreateInfo;
-MinifiExtension* MinifiCreateExtension(const MinifiExtensionCreateInfo* extension_create_info);
-void MinifiConfigureGet(MinifiConfig* config, MinifiStringView key, void(*cb)(void* user_data, MinifiStringView value), void* user_data);
+// api_version is used to provide backwards compatible changes to the MinifiExtensionCreateInfo structure,
+// e.g. if MinifiExtensionCreateInfo gets a new field in version 1.2.0, extensions built with api 1.1.0 do not
+// have to be rebuilt
+MinifiExtension* MinifiCreateExtension(MinifiStringView api_version, const MinifiExtensionCreateInfo*);
+
+MINIFI_OWNED MinifiPublishedMetrics* MinifiPublishedMetricsCreate(size_t count, const MinifiStringView* metric_names, const double* metric_values);
+
+MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext* context, MinifiStringView property_name, MinifiFlowFile* flowfile,
+ void(*cb)(void* user_ctx, MinifiStringView property_value), void* user_ctx);
+void MinifiProcessContextGetProcessorName(MinifiProcessContext* context, void(*cb)(void* user_ctx, MinifiStringView processor_name), void* user_ctx);
+MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext* context, MinifiStringView property_name);
+
+void MinifiLoggerSetMaxLogSize(MinifiLogger*, int32_t);
+void MinifiLoggerLogString(MinifiLogger*, MinifiLogLevel, MinifiStringView);
+MinifiBool MinifiLoggerShouldLog(MinifiLogger*, MinifiLogLevel);
+MinifiLogLevel MinifiLoggerLevel(MinifiLogger*);
+
+MINIFI_OWNED MinifiFlowFile* MinifiProcessSessionGet(MinifiProcessSession*);
+MINIFI_OWNED MinifiFlowFile* MinifiProcessSessionCreate(MinifiProcessSession* session, MinifiFlowFile* parent_flowfile);
+
+void MinifiProcessSessionTransfer(MinifiProcessSession* session, MINIFI_OWNED MinifiFlowFile* flowfile, MinifiStringView relationship_name);
+void MinifiProcessSessionRemove(MinifiProcessSession* session, MINIFI_OWNED MinifiFlowFile* flowfile);
+
+MinifiStatus MinifiProcessSessionRead(MinifiProcessSession*, MinifiFlowFile*, int64_t(*cb)(void* user_ctx, MinifiInputStream*), void* user_ctx);
+MinifiStatus MinifiProcessSessionWrite(MinifiProcessSession*, MinifiFlowFile*, int64_t(*cb)(void* user_ctx, MinifiOutputStream*), void* user_ctx);
+
+void MinifiConfigGet(MinifiConfig* config, MinifiStringView config_key, void(*cb)(void* user_ctx, MinifiStringView config_value), void* user_ctx);
+
+size_t MinifiInputStreamSize(MinifiInputStream*);
+
+int64_t MinifiInputStreamRead(MinifiInputStream* stream, char* buffer, size_t size);
+int64_t MinifiOutputStreamWrite(MinifiOutputStream* stream, const char* data, size_t size);
+
+void MinifiStatusToString(MinifiStatus, void(*cb)(void* user_ctx, MinifiStringView str), void* user_ctx);
+
+void MinifiFlowFileSetAttribute(MinifiProcessSession* session, MinifiFlowFile* flowfile, MinifiStringView attribute_name, const MinifiStringView* attribute_value);
+MinifiBool MinifiFlowFileGetAttribute(MinifiProcessSession* session, MinifiFlowFile* flowfile, MinifiStringView attribute_name,
+ void(*cb)(void* user_ctx, MinifiStringView attribute_value), void* user_ctx);
+void MinifiFlowFileGetAttributes(MinifiProcessSession* session, MinifiFlowFile* flowfile, void(*cb)(void* user_ctx, MinifiStringView attribute_name, MinifiStringView attribute_value), void* user_ctx);
#ifdef __cplusplus
} // extern "C"
diff --git a/minifi-api/include/minifi-cpp/Connection.h b/minifi-api/include/minifi-cpp/Connection.h
index c22c5c1d2..756c05c40 100644
--- a/minifi-api/include/minifi-cpp/Connection.h
+++ b/minifi-api/include/minifi-cpp/Connection.h
@@ -31,7 +31,7 @@
#include "core/Core.h"
#include "minifi-cpp/core/Connectable.h"
#include "minifi-cpp/core/logging/Logger.h"
-#include "core/Relationship.h"
+#include "minifi-cpp/core/Relationship.h"
#include "minifi-cpp/core/FlowFile.h"
#include "minifi-cpp/utils/Literals.h"
diff --git a/minifi-api/include/minifi-cpp/agent/agent_docs.h b/minifi-api/include/minifi-cpp/agent/agent_docs.h
index bccd7160c..833288902 100644
--- a/minifi-api/include/minifi-cpp/agent/agent_docs.h
+++ b/minifi-api/include/minifi-cpp/agent/agent_docs.h
@@ -22,12 +22,11 @@
#include <vector>
#include "minifi-cpp/core/Annotation.h"
+#include "minifi-cpp/core/ControllerServiceApi.h"
#include "minifi-cpp/core/DynamicProperty.h"
-#include "minifi-cpp/core/OutputAttributeDefinition.h"
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+#include "minifi-cpp/core/OutputAttribute.h"
#include "minifi-cpp/core/Property.h"
#include "minifi-cpp/core/Relationship.h"
-#include "minifi-cpp/core/RelationshipDefinition.h"
namespace org::apache::nifi::minifi {
@@ -41,10 +40,10 @@ struct ClassDescription {
std::string full_name_{};
std::string description_{};
std::vector<core::Property> class_properties_{};
- std::span<const core::DynamicProperty> dynamic_properties_{};
+ std::vector<core::DynamicProperty> dynamic_properties_{};
std::vector<core::Relationship> class_relationships_{};
- std::span<const core::OutputAttributeReference> output_attributes_{};
- std::span<const core::ControllerServiceApiDefinition> api_implementations{};
+ std::vector<core::OutputAttribute> output_attributes_{};
+ std::vector<core::ControllerServiceApi> api_implementations{};
bool supports_dynamic_properties_ = false;
bool supports_dynamic_relationships_ = false;
std::string inputRequirement_{};
diff --git a/minifi-api/include/minifi-cpp/core/ConfigurableComponent.h b/minifi-api/include/minifi-cpp/core/ConfigurableComponent.h
index 9f53eba70..1c9196a9e 100644
--- a/minifi-api/include/minifi-cpp/core/ConfigurableComponent.h
+++ b/minifi-api/include/minifi-cpp/core/ConfigurableComponent.h
@@ -23,7 +23,7 @@
#include "Core.h"
#include "Property.h"
-#include "PropertyDefinition.h"
+#include "minifi-cpp/core/PropertyDefinition.h"
#include "utils/expected.h"
namespace org::apache::nifi::minifi::core {
diff --git a/minifi-api/include/minifi-cpp/core/Connectable.h b/minifi-api/include/minifi-cpp/core/Connectable.h
index 15e5205e3..2a1887537 100644
--- a/minifi-api/include/minifi-cpp/core/Connectable.h
+++ b/minifi-api/include/minifi-cpp/core/Connectable.h
@@ -28,7 +28,7 @@
#include "Core.h"
#include <condition_variable>
-#include "Relationship.h"
+#include "minifi-cpp/core/Relationship.h"
#include "Scheduling.h"
#include "minifi-cpp/core/state/FlowIdentifier.h"
#include "minifi-cpp/utils/gsl.h"
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/minifi-api/include/minifi-cpp/core/ControllerServiceApi.h
similarity index 66%
copy from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
copy to minifi-api/include/minifi-cpp/core/ControllerServiceApi.h
index 909ba731d..fe0d450f7 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
+++ b/minifi-api/include/minifi-cpp/core/ControllerServiceApi.h
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -16,17 +16,20 @@
*/
#pragma once
-#include <memory>
#include <string>
-#include "minifi-cpp/utils/Id.h"
-#include "minifi-cpp/core/logging/Logger.h"
+#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
namespace org::apache::nifi::minifi::core {
-struct ProcessorMetadata {
- utils::Identifier uuid;
- std::string name;
- std::shared_ptr<logging::Logger> logger;
+struct ControllerServiceApi {
+ std::string artifact;
+ std::string group;
+ std::string type;
+
+ ControllerServiceApi(const ControllerServiceApiDefinition& definition) // NOLINT(runtime/explicit)
+ : artifact(definition.artifact),
+ group(definition.group),
+ type(definition.type) {}
};
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/DynamicProperty.h b/minifi-api/include/minifi-cpp/core/DynamicProperty.h
index 263326acb..351809122 100644
--- a/minifi-api/include/minifi-cpp/core/DynamicProperty.h
+++ b/minifi-api/include/minifi-cpp/core/DynamicProperty.h
@@ -16,15 +16,22 @@
*/
#pragma once
-#include <string_view>
+#include <string>
+#include "minifi-cpp/core/DynamicPropertyDefinition.h"
namespace org::apache::nifi::minifi::core {
struct DynamicProperty {
- std::string_view name;
- std::string_view value;
- std::string_view description;
+ std::string name;
+ std::string value;
+ std::string description;
bool supports_expression_language = false;
+
+ DynamicProperty(const DynamicPropertyDefinition& definition) // NOLINT(runtime/explicit)
+ : name(definition.name),
+ value(definition.value),
+ description(definition.description),
+ supports_expression_language(definition.supports_expression_language) {}
};
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/FlowFile.h b/minifi-api/include/minifi-cpp/core/FlowFile.h
index 5fa89085c..811f9e8a4 100644
--- a/minifi-api/include/minifi-cpp/core/FlowFile.h
+++ b/minifi-api/include/minifi-cpp/core/FlowFile.h
@@ -23,6 +23,7 @@
#include <set>
#include <unordered_set>
#include <string>
+#include <string_view>
#include <utility>
#include <vector>
@@ -30,6 +31,7 @@
#include "minifi-cpp/core/Connectable.h"
#include "WeakReference.h"
#include "minifi-cpp/utils/FlatMap.h"
+#include "minifi-cpp/core/SpecialFlowAttribute.h"
namespace org::apache::nifi::minifi::core {
@@ -55,11 +57,11 @@ class FlowFile : public virtual CoreComponent, public virtual ReferenceContainer
[[nodiscard]] virtual std::chrono::system_clock::time_point getlineageStartDate() const = 0;
virtual void setLineageStartDate(std::chrono::system_clock::time_point date) = 0;
virtual void setLineageIdentifiers(const std::vector<utils::Identifier>& lineage_Identifiers) = 0;
+ virtual void setAttribute(std::string_view name, std::string value) = 0;
virtual bool getAttribute(std::string_view key, std::string& value) const = 0;
[[nodiscard]] virtual std::optional<std::string> getAttribute(std::string_view key) const = 0;
virtual bool updateAttribute(std::string_view key, const std::string& value) = 0;
virtual bool removeAttribute(std::string_view key) = 0;
- virtual bool setAttribute(std::string_view key, std::string value) = 0;
[[nodiscard]] virtual std::map<std::string, std::string> getAttributes() const = 0;
virtual AttributeMap *getAttributesPtr() = 0;
virtual bool addAttribute(std::string_view key, const std::string& value) = 0;
@@ -79,22 +81,4 @@ class FlowFile : public virtual CoreComponent, public virtual ReferenceContainer
static std::shared_ptr<FlowFile> create();
};
-struct SpecialFlowAttribute {
- MINIFIAPI static constexpr std::string_view PATH = "path";
- MINIFIAPI static constexpr std::string_view ABSOLUTE_PATH = "absolute.path";
- MINIFIAPI static constexpr std::string_view FILENAME = "filename";
- MINIFIAPI static constexpr std::string_view UUID = "uuid";
- MINIFIAPI static constexpr std::string_view priority = "priority";
- MINIFIAPI static constexpr std::string_view MIME_TYPE = "mime.type";
- MINIFIAPI static constexpr std::string_view DISCARD_REASON = "discard.reason";
- MINIFIAPI static constexpr std::string_view ALTERNATE_IDENTIFIER = "alternate.identifier";
- MINIFIAPI static constexpr std::string_view FLOW_ID = "flow.id";
-
- static constexpr std::array<std::string_view, 9> getSpecialFlowAttributes() {
- return {
- PATH, ABSOLUTE_PATH, FILENAME, UUID, priority, MIME_TYPE, DISCARD_REASON, ALTERNATE_IDENTIFIER, FLOW_ID
- };
- }
-};
-
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h b/minifi-api/include/minifi-cpp/core/OutputAttribute.h
similarity index 61%
rename from minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
rename to minifi-api/include/minifi-cpp/core/OutputAttribute.h
index 909ba731d..6adb00ec3 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorMetadata.h
+++ b/minifi-api/include/minifi-cpp/core/OutputAttribute.h
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -16,17 +16,25 @@
*/
#pragma once
-#include <memory>
#include <string>
-#include "minifi-cpp/utils/Id.h"
-#include "minifi-cpp/core/logging/Logger.h"
+#include <vector>
+
+#include "minifi-cpp/core/Relationship.h"
+#include "minifi-cpp/core/OutputAttributeDefinition.h"
namespace org::apache::nifi::minifi::core {
-struct ProcessorMetadata {
- utils::Identifier uuid;
+struct OutputAttribute {
std::string name;
- std::shared_ptr<logging::Logger> logger;
+ std::vector<Relationship> relationships;
+ std::string description;
+
+ OutputAttribute() = default;
+
+ OutputAttribute(const OutputAttributeReference& reference) // NOLINT(runtime/explicit)
+ : name(reference.name),
+ relationships(reference.relationships.begin(), reference.relationships.end()),
+ description(reference.description) {}
};
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/ProcessContext.h b/minifi-api/include/minifi-cpp/core/ProcessContext.h
index c9ba38fc5..a81e45e0f 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessContext.h
+++ b/minifi-api/include/minifi-cpp/core/ProcessContext.h
@@ -50,8 +50,6 @@ class ProcessContext : public virtual core::VariableRegistry, public virtual uti
virtual const ProcessorInfo& getProcessorInfo() const = 0;
virtual Processor& getProcessor() const = 0;
- virtual bool hasNonEmptyProperty(std::string_view name) const = 0;
-
virtual nonstd::expected<std::string, std::error_code> getProperty(std::string_view name, const FlowFile* flow_file = nullptr) const = 0;
nonstd::expected<std::string, std::error_code> getProperty(const Property& property, const FlowFile* flow_file = nullptr) const { return getProperty(property.getName(), flow_file); }
nonstd::expected<std::string, std::error_code> getProperty(const PropertyReference& property_reference, const FlowFile* flow_file = nullptr) const {
@@ -81,13 +79,15 @@ class ProcessContext : public virtual core::VariableRegistry, public virtual uti
virtual bool isRunning() const = 0;
virtual bool isAutoTerminated(Relationship relationship) const = 0;
virtual uint8_t getMaxConcurrentTasks() const = 0;
- virtual void yield() = 0;
virtual std::shared_ptr<core::Repository> getProvenanceRepository() = 0;
virtual std::shared_ptr<core::ContentRepository> getContentRepository() const = 0;
virtual std::shared_ptr<core::Repository> getFlowFileRepository() const = 0;
virtual bool hasIncomingConnections() const = 0;
+ virtual bool hasNonEmptyProperty(std::string_view name) const = 0;
+ virtual void yield() = 0;
+
virtual std::shared_ptr<core::controller::ControllerService> getControllerService(const std::string &identifier, const utils::Identifier &processor_uuid) const = 0;
static constexpr char const* DefaultStateStorageName = "defaultstatestorage";
diff --git a/minifi-api/include/minifi-cpp/core/ProcessSession.h b/minifi-api/include/minifi-cpp/core/ProcessSession.h
index 052d1eaf2..d751515a0 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessSession.h
+++ b/minifi-api/include/minifi-cpp/core/ProcessSession.h
@@ -48,11 +48,10 @@ class ProcessSession : public virtual ReferenceContainer {
virtual std::shared_ptr<provenance::ProvenanceReporter> getProvenanceReporter() = 0;
virtual void flushContent() = 0;
virtual std::shared_ptr<core::FlowFile> get() = 0;
- virtual std::shared_ptr<core::FlowFile> create(const core::FlowFile* const parent = nullptr) = 0;
+ virtual std::shared_ptr<core::FlowFile> create(const core::FlowFile* parent = nullptr) = 0;
virtual void add(const std::shared_ptr<core::FlowFile> &record) = 0;
virtual std::shared_ptr<core::FlowFile> clone(const core::FlowFile& parent) = 0;
virtual std::shared_ptr<core::FlowFile> clone(const core::FlowFile& parent, int64_t offset, int64_t size) = 0;
- virtual void transfer(const std::shared_ptr<core::FlowFile>& flow, const Relationship& relationship) = 0;
virtual void transferToCustomRelationship(const std::shared_ptr<core::FlowFile>& flow, const std::string& relationship_name) = 0;
virtual void putAttribute(core::FlowFile& flow, std::string_view key, const std::string& value) = 0;
@@ -117,6 +116,8 @@ class ProcessSession : public virtual ReferenceContainer {
virtual bool existsFlowFileInRelationship(const Relationship &relationship) = 0;
virtual bool hasBeenTransferred(const core::FlowFile &flow) const = 0;
+
+ virtual void transfer(const std::shared_ptr<FlowFile>& ff, const Relationship& relationship) = 0;
};
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorApi.h b/minifi-api/include/minifi-cpp/core/ProcessorApi.h
index ae39feae5..2ce94f853 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorApi.h
+++ b/minifi-api/include/minifi-cpp/core/ProcessorApi.h
@@ -19,14 +19,9 @@
#include <memory>
#include <string>
-#include "ConfigurableComponent.h"
#include "Property.h"
-#include "DynamicProperty.h"
-#include "Core.h"
#include "minifi-cpp/core/Annotation.h"
-#include "Scheduling.h"
-#include "minifi-cpp/core/state/nodes/MetricsBase.h"
-#include "minifi-cpp/utils/gsl.h"
+#include "minifi-cpp/core/FlowFile.h"
#include "minifi-cpp/core/logging/Logger.h"
#include "minifi-cpp/core/ProcessorMetricsExtension.h"
@@ -63,7 +58,7 @@ class ProcessorApi {
virtual void notifyStop() = 0;
virtual annotation::Input getInputRequirement() const = 0;
virtual std::shared_ptr<ProcessorMetricsExtension> getMetricsExtension() const = 0;
- virtual void setLoggerCallback(const std::function<void(logging::LOG_LEVEL level, const std::string& message)>& callback) = 0;
+ virtual void forEachLogger(const std::function<void(std::shared_ptr<logging::Logger>)>& callback) = 0;
};
} // namespace core
diff --git a/minifi-api/include/minifi-cpp/core/ProcessorDescriptor.h b/minifi-api/include/minifi-cpp/core/ProcessorDescriptor.h
index e719b9252..9ebc2a384 100644
--- a/minifi-api/include/minifi-cpp/core/ProcessorDescriptor.h
+++ b/minifi-api/include/minifi-cpp/core/ProcessorDescriptor.h
@@ -28,6 +28,7 @@ class ProcessorDescriptor {
virtual void setSupportedRelationships(std::span<const RelationshipDefinition> relationships) = 0;
virtual void setSupportedProperties(std::span<const PropertyReference> properties) = 0;
+ virtual void setSupportedProperties(std::span<const Property> properties) = 0;
};
} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-cpp/core/Property.h b/minifi-api/include/minifi-cpp/core/Property.h
index 68f09d975..e4fb1f986 100644
--- a/minifi-api/include/minifi-cpp/core/Property.h
+++ b/minifi-api/include/minifi-cpp/core/Property.h
@@ -24,8 +24,8 @@
#include <utility>
#include <vector>
-#include "PropertyDefinition.h"
-#include "PropertyValidator.h"
+#include "minifi-cpp/core/PropertyDefinition.h"
+#include "minifi-cpp/core/PropertyValidator.h"
#include "range/v3/range/conversion.hpp"
#include "range/v3/view/transform.hpp"
#include "minifi-cpp/utils/gsl.h"
@@ -70,9 +70,6 @@ class Property final {
bool isSensitive() const;
bool supportsExpressionLanguage() const;
std::vector<std::string> getValues();
- PropertyReference getReference() const {
- return PropertyReference(name_, display_name_, description_, is_required_, is_sensitive_, {}, {}, default_value_, validator_, supports_el_);
- }
void setSupportsExpressionLanguage(bool supportEl);
diff --git a/minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h b/minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h
index acab5ce10..38c069356 100644
--- a/minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h
+++ b/minifi-api/include/minifi-cpp/core/logging/LoggerFactory.h
@@ -21,7 +21,7 @@
#include <string_view>
#include <memory>
-#include "Logger.h"
+#include "minifi-cpp/core/logging/Logger.h"
#include "minifi-cpp/core/Core.h"
namespace org::apache::nifi::minifi::core::logging {
diff --git a/minifi-api/include/minifi-cpp/io/ArchiveStream.h b/minifi-api/include/minifi-cpp/io/ArchiveStream.h
index 2048f5fdd..e4e1b9b8c 100644
--- a/minifi-api/include/minifi-cpp/io/ArchiveStream.h
+++ b/minifi-api/include/minifi-cpp/io/ArchiveStream.h
@@ -21,8 +21,8 @@
#include <memory>
#include <string>
-#include "OutputStream.h"
-#include "InputStream.h"
+#include "io/OutputStream.h"
+#include "io/InputStream.h"
#include "core/Core.h"
#include "minifi-cpp/core/logging/Logger.h"
diff --git a/minifi-api/include/minifi-cpp/io/BaseStream.h b/minifi-api/include/minifi-cpp/io/BaseStream.h
index f6a5595c8..51da4f1fd 100644
--- a/minifi-api/include/minifi-cpp/io/BaseStream.h
+++ b/minifi-api/include/minifi-cpp/io/BaseStream.h
@@ -23,8 +23,8 @@
#include <iostream>
#include <cstdint>
#include "core/expect.h"
-#include "InputStream.h"
-#include "OutputStream.h"
+#include "io/InputStream.h"
+#include "io/OutputStream.h"
namespace org::apache::nifi::minifi::io {
diff --git a/minifi-api/include/minifi-cpp/utils/TimeUtil.h b/minifi-api/include/minifi-cpp/utils/TimeUtil.h
index 6969261e6..5c1cc2cf0 100644
--- a/minifi-api/include/minifi-cpp/utils/TimeUtil.h
+++ b/minifi-api/include/minifi-cpp/utils/TimeUtil.h
@@ -17,29 +17,37 @@
#pragma once
-#include <string>
#include <chrono>
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <string>
namespace org::apache::nifi::minifi::utils::timeutils {
class Clock {
public:
virtual ~Clock() = default;
- virtual std::chrono::milliseconds timeSinceEpoch() const = 0;
- virtual bool wait_until(std::condition_variable& cv, std::unique_lock<std::mutex>& lck, std::chrono::milliseconds time, const std::function<bool()>& pred) {
+ Clock() = default;
+ Clock(const Clock&) = delete;
+ Clock(Clock&&) = delete;
+ Clock& operator=(const Clock&) = delete;
+ Clock& operator=(Clock&&) = delete;
+
+ [[nodiscard]] virtual std::chrono::milliseconds timeSinceEpoch() const = 0;
+ virtual bool wait_until(std::condition_variable& cv, std::unique_lock<std::mutex>& lck, std::chrono::milliseconds time,
+ const std::function<bool()>& pred) {
return cv.wait_for(lck, time - timeSinceEpoch(), pred);
}
};
class SteadyClock : public Clock {
public:
- std::chrono::milliseconds timeSinceEpoch() const override {
+ [[nodiscard]] std::chrono::milliseconds timeSinceEpoch() const override {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch());
}
- virtual std::chrono::time_point<std::chrono::steady_clock> now() const {
- return std::chrono::steady_clock::now();
- }
+ [[nodiscard]] virtual std::chrono::time_point<std::chrono::steady_clock> now() const { return std::chrono::steady_clock::now(); }
};
std::shared_ptr<SteadyClock> getClock();
@@ -52,4 +60,3 @@ void dateSetGlobalInstall(const std::string& install);
#endif
} // namespace org::apache::nifi::minifi::utils::timeutils
-
diff --git a/minifi-api/minifi-c-api.def b/minifi-api/minifi-c-api.def
new file mode 100644
index 000000000..18fad4bff
--- /dev/null
+++ b/minifi-api/minifi-c-api.def
@@ -0,0 +1,25 @@
+LIBRARY core-minifi.dll
+EXPORTS
+ MinifiCreateExtension
+ MinifiPublishedMetricsCreate
+ MinifiProcessContextGetProperty
+ MinifiProcessContextGetProcessorName
+ MinifiProcessContextHasNonEmptyProperty
+ MinifiLoggerSetMaxLogSize
+ MinifiLoggerLogString
+ MinifiLoggerShouldLog
+ MinifiLoggerLevel
+ MinifiProcessSessionGet
+ MinifiProcessSessionCreate
+ MinifiProcessSessionTransfer
+ MinifiProcessSessionRemove
+ MinifiProcessSessionRead
+ MinifiProcessSessionWrite
+ MinifiConfigGet
+ MinifiInputStreamSize
+ MinifiInputStreamRead
+ MinifiOutputStreamWrite
+ MinifiStatusToString
+ MinifiFlowFileSetAttribute
+ MinifiFlowFileGetAttribute
+ MinifiFlowFileGetAttributes
\ No newline at end of file
diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp
index 121c26df8..26983d028 100644
--- a/minifi_main/AgentDocs.cpp
+++ b/minifi_main/AgentDocs.cpp
@@ -80,8 +80,8 @@ std::string formatDescription(const minifi::core::DynamicProperty& dynamic_prope
return formatDescription(dynamic_property.description, false, dynamic_property.supports_expression_language);
}
-std::string formatListOfRelationships(std::span<const minifi::core::RelationshipDefinition> relationships) {
- return minifi::utils::string::join(", ", relationships, [](const auto& relationship) { return relationship.name; });
+std::string formatListOfRelationships(std::span<const minifi::core::Relationship> relationships) {
+ return minifi::utils::string::join(", ", relationships, [](const auto& relationship) { return relationship.getName(); });
}
inline constexpr std::string_view APACHE_LICENSE = R"license(<!--