You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ph...@apache.org on 2017/11/02 23:36:58 UTC
[9/9] nifi-minifi-cpp git commit: MINIFICPP-110 Add ExecuteScript
processor with support for Python and Lua scripting
MINIFICPP-110 Add ExecuteScript processor with support for Python and Lua scripting
This closes #163.
Signed-off-by: Marc Parisi <ph...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/commit/9a10b98e
Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/tree/9a10b98e
Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/diff/9a10b98e
Branch: refs/heads/master
Commit: 9a10b98e145ee92550f54c01a1129de0dad78653
Parents: 726dc40
Author: Andy I. Christianson <an...@andyic.org>
Authored: Tue Sep 26 15:44:17 2017 -0400
Committer: Marc Parisi <ph...@apache.org>
Committed: Thu Nov 2 19:33:05 2017 -0400
----------------------------------------------------------------------
.travis.yml | 12 +-
CMakeLists.txt | 8 +
LICENSE | 62 +
README.md | 34 +-
extensions/script/CMakeLists.txt | 117 +
extensions/script/ExecuteScript.cpp | 165 +
extensions/script/ExecuteScript.h | 97 +
extensions/script/ScriptEngine.h | 52 +
extensions/script/ScriptFlowFile.cpp | 64 +
extensions/script/ScriptFlowFile.h | 49 +
extensions/script/ScriptProcessContext.cpp | 44 +
extensions/script/ScriptProcessContext.h | 48 +
extensions/script/lua/LuaBaseStream.cpp | 73 +
extensions/script/lua/LuaBaseStream.h | 58 +
extensions/script/lua/LuaProcessSession.cpp | 75 +
extensions/script/lua/LuaProcessSession.h | 88 +
extensions/script/lua/LuaScriptEngine.cpp | 68 +
extensions/script/lua/LuaScriptEngine.h | 83 +
extensions/script/python/PyBaseStream.cpp | 68 +
extensions/script/python/PyBaseStream.h | 52 +
extensions/script/python/PyProcessSession.cpp | 77 +
extensions/script/python/PyProcessSession.h | 89 +
extensions/script/python/PythonBindings.h | 77 +
extensions/script/python/PythonScriptEngine.cpp | 84 +
extensions/script/python/PythonScriptEngine.h | 118 +
libminifi/test/script-tests/CMakeLists.txt | 64 +
.../test/script-tests/ExecuteScriptTests.cpp | 640 +
.../test/script-tests/LuaExecuteScriptTests.cpp | 338 +
.../script-tests/PythonExecuteScriptTests.cpp | 325 +
thirdparty/pybind11/.gitignore | 37 +
thirdparty/pybind11/include/pybind11/attr.h | 489 +
.../pybind11/include/pybind11/buffer_info.h | 108 +
thirdparty/pybind11/include/pybind11/cast.h | 2053 +++
thirdparty/pybind11/include/pybind11/chrono.h | 162 +
thirdparty/pybind11/include/pybind11/common.h | 2 +
thirdparty/pybind11/include/pybind11/complex.h | 61 +
.../pybind11/include/pybind11/detail/class.h | 606 +
.../pybind11/include/pybind11/detail/common.h | 800 +
.../pybind11/include/pybind11/detail/descr.h | 185 +
.../pybind11/include/pybind11/detail/init.h | 325 +
.../include/pybind11/detail/internals.h | 247 +
.../pybind11/include/pybind11/detail/typeid.h | 53 +
thirdparty/pybind11/include/pybind11/eigen.h | 610 +
thirdparty/pybind11/include/pybind11/embed.h | 194 +
thirdparty/pybind11/include/pybind11/eval.h | 117 +
.../pybind11/include/pybind11/functional.h | 85 +
thirdparty/pybind11/include/pybind11/iostream.h | 200 +
thirdparty/pybind11/include/pybind11/numpy.h | 1600 ++
.../pybind11/include/pybind11/operators.h | 168 +
thirdparty/pybind11/include/pybind11/options.h | 65 +
thirdparty/pybind11/include/pybind11/pybind11.h | 1965 +++
thirdparty/pybind11/include/pybind11/pytypes.h | 1332 ++
thirdparty/pybind11/include/pybind11/stl.h | 369 +
thirdparty/pybind11/include/pybind11/stl_bind.h | 599 +
thirdparty/sol2-2.17.5/sol.hpp | 14872 +++++++++++++++++
55 files changed, 30430 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 35a2429..51173c5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,12 +33,16 @@ matrix:
- openssl
- libcurl4-openssl-dev
- ccache
+ - libpython3.4-dev
+ - liblua5.1-0-dev
before_install:
# Establish updated toolchain as default
- sudo unlink /usr/bin/gcc && sudo ln -s /usr/bin/gcc-4.8 /usr/bin/gcc
- sudo unlink /usr/bin/g++ && sudo ln -s /usr/bin/g++-4.8 /usr/bin/g++
- os: osx
osx_image: xcode7.3
+ env:
+ - CMAKE_BUILD_OPTIONS="-DENABLE_LUA_SCRIPTING=1"
# https://docs.travis-ci.com/user/osx-ci-environment/#Homebrew
before_install:
- brew update
@@ -49,8 +53,12 @@ matrix:
- package='openssl'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='doxygen'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='graphviz'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
+ - package='python'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
+ - package='lua'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- os: osx
osx_image: xcode8.3
+ env:
+ - CMAKE_BUILD_OPTIONS="-DENABLE_LUA_SCRIPTING=1"
# https://docs.travis-ci.com/user/osx-ci-environment/#Homebrew
before_install:
- brew update
@@ -61,7 +69,9 @@ matrix:
- package='openssl'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='doxygen'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='graphviz'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
+ - package='python'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
+ - package='lua'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
script:
# Travis instances typically have ~2 vcpus. Parallel testing is currently BROKEN due to socket tests; do not use.
- - mkdir ./build && cd ./build && cmake .. && make -j2 VERBOSE=1 && make test ARGS="--output-on-failure" && make linter && make apache-rat && make docs
+ - mkdir ./build && cd ./build && cmake .. ${CMAKE_BUILD_OPTIONS} && make -j2 VERBOSE=1 && make test ARGS="--output-on-failure" && make linter && make apache-rat && make docs
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b0c6853..07fabcf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -148,6 +148,14 @@ createExtension(DISABLE_LIBARCHIVE
BUILD_TP
"thirdparty/libarchive-3.3.2")
+## Scripting extensions
+createExtension(DISABLE_SCRIPTING
+ SCRIPTING-EXTENSIONS
+ "SCRIPTING EXTENSIONS"
+ "This enables scripting"
+ "extensions/script"
+ "${TEST_DIR}/script-tests")
+
## NOW WE CAN ADD LIBRARIES AND EXTENSIONS TO MAIN
add_subdirectory(main)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 2f34ab8..815dafc 100644
--- a/LICENSE
+++ b/LICENSE
@@ -426,6 +426,68 @@ FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
+This product bundles 'sol2' which is available under The MIT License.
+
+The MIT License (MIT)
+
+Copyright (c) 2013-2017 Rapptz, ThePhD, and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'pybind11' which is available under a BSD-style license.
+
+Copyright (c) 2016 Wenzel Jakob <we...@epfl.ch>, All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You are under no obligation whatsoever to provide any bug fixes, patches, or
+upgrades to the features, functionality or performance of the source code
+("Enhancements") to anyone; however, if you choose to make your Enhancements
+available either publicly, or directly to the author of this software, without
+imposing a separate written license agreement for such Enhancements, then you
+hereby grant the following license: a non-exclusive, royalty-free perpetual
+license to install, use, modify, prepare derivative works, incorporate into
+other computer software, distribute, and sublicense such enhancements or
+derivative works thereof, in binary and source code form.
+
This product bundles 'CivetWeb' which is available under a MIT license.
Copyright (c) 2013-2015 The CivetWeb developers (CREDITS.md)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index cc833e0..0c73b2b 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,7 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a
* The processors currently implemented include:
* AppendHostInfo
* ExecuteProcess
+ * ExecuteScript
* GetFile
* GenerateFlowFile
* InvokeHTTP
@@ -74,6 +75,9 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a
* 4.8.4 or greater
* g++
* 4.8.4 or greater
+
+**NOTE** if Lua support is enabled, then a C++ compiler with support for c++-14 must be used. If using GCC, version 6.x
+or greater is recommended.
#### Libraries / Development Headers
* libboost and boost-devel
@@ -82,6 +86,8 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a
* librocksdb4.1 and librocksdb-dev
* libuuid and uuid-dev
* openssl
+* Python 3 and development headers -- Required, unless Python support is disabled
+* Lua and development headers -- Optional, unless Lua support is enabled
** NOTE: IF ROCKSDB IS NOT INSTALLED, IT WILL BE BUILT FROM THE THIRD PARTY
@@ -95,20 +101,30 @@ DIRECTORY UNLESS YOU SPECIFY -DDISABLE_ROCKSDB=true WITH CMAKE ***
* libcurl
* libssl and libcrypto from openssl
* libarchive
+* Python 3 -- Required, unless Python support is disabled
+* Lua -- Optional, unless Lua support is enabled
The needed dependencies can be installed with the following commands for:
Yum based Linux Distributions
+
+**NOTE** if a newer compiler is required, such as when Lua support is enabled, it is recommended to use a newer compiler
+using a devtools-* package from the Software Collections (SCL).
+
```
# ~/Development/code/apache/nifi-minifi-cpp on git:master
$ yum install cmake \
gcc gcc-c++ \
libcurl-devel \
- rocksdb-dev rocksdb \
+ rocksdb-devel rocksdb \
libuuid libuuid-devel \
boost-devel \
- libssl-dev \
+ openssl-devel \
doxygen
+$ # (Optional) for building Python support
+$ yum install python34-devel
+$ # (Optional) for building Lua support
+$ yum install lua-devel
$ # (Optional) for building docker image
$ yum install docker
$ # (Optional) for system integration tests
@@ -126,6 +142,10 @@ $ apt-get install cmake \
uuid-dev uuid \
libboost-all-dev libssl-dev \
doxygen
+$ # (Optional) for building Python support
+$ apt-get install libpython3-dev
+$ # (Optional) for building Lua support
+$ apt-get install liblua5.1-0-dev
$ # (Optional) for building docker image
$ apt-get install docker.io
$ # (Optional) for system integration tests
@@ -140,6 +160,8 @@ $ brew install cmake \
ossp-uuid \
boost \
openssl \
+ python \
+ lua \
doxygen
$ brew install curl
$ brew link curl --force
@@ -162,6 +184,14 @@ $ sudo pip install virtualenv
```
- Perform a `cmake ..` to generate the project files
+ - Optionally, disable or enable features using any combination of the following flags (more information is available
+ on the [wiki](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=74685143)):
+ - `-DDISABLE_CURL=1`
+ - `-DDISABLE_ROCKSDB=1`
+ - `-DDISABLE_LIBARCHIVE=1`
+ - `-DDISABLE_SCRIPTING=1`
+ - `-DDISABLE_PYTHON_SCRIPTING=1`
+ - `-DENABLE_LUA_SCRIPTING=1`
```
# ~/Development/code/apache/nifi-minifi-cpp on git:master
$ cmake ..
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/extensions/script/CMakeLists.txt b/extensions/script/CMakeLists.txt
new file mode 100644
index 0000000..11f38df
--- /dev/null
+++ b/extensions/script/CMakeLists.txt
@@ -0,0 +1,117 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++14 " COMPILER_SUPPORTS_CXX14)
+CHECK_CXX_COMPILER_FLAG("-std=c++1y " COMPILER_SUPPORTS_CXX1Y)
+
+if (ENABLE_LUA_SCRIPTING)
+ if(COMPILER_SUPPORTS_CXX14)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
+ elseif(COMPILER_SUPPORTS_CXX1Y)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y")
+ else()
+ message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++14 support and lua is enabled. Please use a different C++ compiler or disable lua.")
+ endif()
+
+ set(CMAKE_CXX_STANDARD 14)
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols")
+set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols")
+
+include_directories(../../libminifi/include ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/)
+
+file(GLOB SOURCES "*.cpp")
+
+add_library(minifi-script-extensions STATIC ${SOURCES})
+set_property(TARGET minifi-script-extensions PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(THREADS_HAVE_PTHREAD_ARG)
+ target_compile_options(PUBLIC minifi-script-extensions "-pthread")
+endif()
+if(CMAKE_THREAD_LIBS_INIT)
+ target_link_libraries(minifi-script-extensions "${CMAKE_THREAD_LIBS_INIT}")
+endif()
+
+find_package(UUID REQUIRED)
+target_link_libraries(minifi-script-extensions ${LIBMINIFI} ${UUID_LIBRARIES} ${JSONCPP_LIB})
+add_dependencies(minifi-script-extensions jsoncpp_project)
+find_package(OpenSSL REQUIRED)
+include_directories(${OPENSSL_INCLUDE_DIR})
+target_link_libraries(minifi-script-extensions ${CMAKE_DL_LIBS})
+
+if (NOT DISABLE_PYTHON_SCRIPTING)
+ find_package(PythonLibs REQUIRED)
+
+ include_directories(${PYTHON_INCLUDE_DIR})
+ include_directories(../../thirdparty/pybind11/include)
+
+ include_directories(python)
+ add_definitions(-DPYTHON_SUPPORT)
+
+ file(GLOB PY_SOURCES "python/*.cpp")
+ add_library(minifi-python-extensions STATIC ${PY_SOURCES})
+
+ target_link_libraries(minifi-python-extensions ${PYTHON_LIBRARIES})
+ target_link_libraries(minifi-script-extensions minifi-python-extensions)
+endif()
+
+if (ENABLE_LUA_SCRIPTING)
+ find_package(Lua REQUIRED)
+
+ include_directories(${LUA_INCLUDE_DIR})
+ include_directories(../../thirdparty/sol2-2.17.5)
+
+ include_directories(lua)
+ add_definitions(-DLUA_SUPPORT)
+
+ file(GLOB LUA_SOURCES "lua/*.cpp")
+ add_library(minifi-lua-extensions STATIC ${LUA_SOURCES})
+
+ target_link_libraries(minifi-lua-extensions ${LUA_LIBRARIES})
+ target_link_libraries(minifi-script-extensions minifi-lua-extensions)
+endif()
+
+find_package(ZLIB REQUIRED)
+include_directories(${ZLIB_INCLUDE_DIRS})
+target_link_libraries (minifi-script-extensions ${ZLIB_LIBRARIES})
+find_package(Boost COMPONENTS system filesystem REQUIRED)
+include_directories(${Boost_INCLUDE_DIRS})
+target_link_libraries(minifi-script-extensions ${Boost_SYSTEM_LIBRARY})
+target_link_libraries(minifi-script-extensions ${Boost_FILESYSTEM_LIBRARY})
+
+if (WIN32)
+ set_target_properties(minifi-script-extensions PROPERTIES
+ LINK_FLAGS "/WHOLEARCHIVE"
+ )
+elseif (APPLE)
+ set_target_properties(minifi-script-extensions PROPERTIES
+ LINK_FLAGS "-Wl,-all_load"
+ )
+else ()
+ set_target_properties(minifi-script-extensions PROPERTIES
+ LINK_FLAGS "-Wl,--whole-archive"
+ )
+endif ()
+
+SET (SCRIPTING-EXTENSIONS minifi-script-extensions PARENT_SCOPE)
+
+register_extension(minifi-script-extensions)
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ExecuteScript.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/ExecuteScript.cpp b/extensions/script/ExecuteScript.cpp
new file mode 100644
index 0000000..5590d30
--- /dev/null
+++ b/extensions/script/ExecuteScript.cpp
@@ -0,0 +1,165 @@
+/**
+ * @file ExecuteScript.cpp
+
+ * ExecuteScript class implementation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <set>
+#include <utility>
+
+#ifdef PYTHON_SUPPORT
+#include <PythonScriptEngine.h>
+#endif // PYTHON_SUPPORT
+
+#ifdef LUA_SUPPORT
+#include <LuaScriptEngine.h>
+#endif // LUA_SUPPORT
+
+#include "ExecuteScript.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+core::Property ExecuteScript::ScriptEngine("Script Engine", // NOLINT
+ "The engine to execute scripts (python, lua)", "python");
+core::Property ExecuteScript::ScriptFile("Script File", // NOLINT
+ R"(Path to script file to execute.
+ Only one of Script File or Script Body may be used)", "");
+core::Property ExecuteScript::ScriptBody("Script Body", // NOLINT
+ R"(Body of script to execute.
+ Only one of Script File or Script Body may be used)", "");
+core::Property ExecuteScript::ModuleDirectory("Module Directory", // NOLINT
+ R"(Comma-separated list of paths to files and/or directories which
+ contain modules required by the script)", "");
+
+core::Relationship ExecuteScript::Success("success", "Script successes"); // NOLINT
+core::Relationship ExecuteScript::Failure("failure", "Script failures"); // NOLINT
+
+void ExecuteScript::initialize() {
+ std::set<core::Property> properties;
+ properties.insert(ScriptEngine);
+ properties.insert(ScriptFile);
+ properties.insert(ScriptBody);
+ properties.insert(ModuleDirectory);
+ setSupportedProperties(std::move(properties));
+
+ std::set<core::Relationship> relationships;
+ relationships.insert(Success);
+ relationships.insert(Failure);
+ setSupportedRelationships(std::move(relationships));
+
+#ifdef PYTHON_SUPPORT
+ python::PythonScriptEngine::initialize();
+#endif // PYTHON_SUPPORT
+}
+
+void ExecuteScript::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) {
+ if (!context->getProperty(ScriptEngine.getName(), script_engine_)) {
+ logger_->log_error("Script Engine attribute is missing or invalid");
+ }
+
+ context->getProperty(ScriptFile.getName(), script_file_);
+ context->getProperty(ScriptBody.getName(), script_body_);
+ context->getProperty(ModuleDirectory.getName(), module_directory_);
+
+ if (script_file_.empty() && script_engine_.empty()) {
+ logger_->log_error("Either Script Body or Script File must be defined");
+ return;
+ }
+}
+
+void ExecuteScript::onTrigger(const std::shared_ptr<core::ProcessContext> &context,
+ const std::shared_ptr<core::ProcessSession> &session) {
+ try {
+ std::shared_ptr<script::ScriptEngine> engine;
+
+ // Use an existing engine, if one is available
+ if (script_engine_q_.try_dequeue(engine)) {
+ logger_->log_info("Using available %s script engine instance", script_engine_);
+ } else {
+ logger_->log_info("Creating new %s script instance", script_engine_);
+ logger_->log_info("Approximately %d %s script instances created for this processor",
+ script_engine_q_.size_approx(),
+ script_engine_);
+
+ if (script_engine_ == "python") {
+#ifdef PYTHON_SUPPORT
+ engine = createEngine<python::PythonScriptEngine>();
+#else
+ throw std::runtime_error("Python support is disabled in this build.");
+#endif // PYTHON_SUPPORT
+ } else if (script_engine_ == "lua") {
+#ifdef LUA_SUPPORT
+ engine = createEngine<lua::LuaScriptEngine>();
+#else
+ throw std::runtime_error("Lua support is disabled in this build.");
+#endif // LUA_SUPPORT
+ }
+
+ if (engine == nullptr) {
+ throw std::runtime_error("No script engine available");
+ }
+
+ if (!script_body_.empty()) {
+ engine->eval(script_body_);
+ } else if (!script_file_.empty()) {
+ engine->evalFile(script_file_);
+ } else {
+ throw std::runtime_error("Neither Script Body nor Script File is available to execute");
+ }
+ }
+
+ if (script_engine_ == "python") {
+#ifdef PYTHON_SUPPORT
+ triggerEngineProcessor<python::PythonScriptEngine>(engine, context, session);
+#else
+ throw std::runtime_error("Python support is disabled in this build.");
+#endif // PYTHON_SUPPORT
+ } else if (script_engine_ == "lua") {
+#ifdef LUA_SUPPORT
+ triggerEngineProcessor<lua::LuaScriptEngine>(engine, context, session);
+#else
+ throw std::runtime_error("Lua support is disabled in this build.");
+#endif // LUA_SUPPORT
+ }
+
+ // Make engine available for use again
+ if (script_engine_q_.size_approx() < getMaxConcurrentTasks()) {
+ logger_->log_info("Releasing %s script engine", script_engine_);
+ script_engine_q_.enqueue(engine);
+ } else {
+ logger_->log_info("Destroying script engine because it is no longer needed");
+ }
+ } catch (std::exception &exception) {
+ logger_->log_error("Caught Exception %s", exception.what());
+ this->yield();
+ } catch (...) {
+ logger_->log_error("Caught Exception");
+ this->yield();
+ }
+}
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ExecuteScript.h
----------------------------------------------------------------------
diff --git a/extensions/script/ExecuteScript.h b/extensions/script/ExecuteScript.h
new file mode 100644
index 0000000..441969a
--- /dev/null
+++ b/extensions/script/ExecuteScript.h
@@ -0,0 +1,97 @@
+/**
+ * @file ExecuteScript.h
+ * ExecuteScript class declaration
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_EXECUTESCRIPT_H
+#define NIFI_MINIFI_CPP_EXECUTESCRIPT_H
+
+#include <concurrentqueue.h>
+#include <core/Resource.h>
+#include <core/Processor.h>
+
+#include "ScriptEngine.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+class ExecuteScript : public core::Processor {
+ public:
+ explicit ExecuteScript(const std::string &name, uuid_t uuid = nullptr)
+ : Processor(name, uuid),
+ logger_(logging::LoggerFactory<ExecuteScript>::getLogger()),
+ script_engine_q_() {
+ }
+
+ static core::Property ScriptEngine;
+ static core::Property ScriptFile;
+ static core::Property ScriptBody;
+ static core::Property ModuleDirectory;
+
+ static core::Relationship Success;
+ static core::Relationship Failure;
+
+ void initialize() override;
+ void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override;
+ void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+ logger_->log_error("onTrigger invocation with raw pointers is not implemented");
+ }
+ void onTrigger(const std::shared_ptr<core::ProcessContext> &context, const std::shared_ptr<core::ProcessSession> &session) override;
+
+ private:
+ std::shared_ptr<logging::Logger> logger_;
+
+ std::string script_engine_;
+ std::string script_file_;
+ std::string script_body_;
+ std::string module_directory_;
+
+ moodycamel::ConcurrentQueue<std::shared_ptr<script::ScriptEngine>> script_engine_q_;
+
+ template<typename T>
+ std::shared_ptr<T> createEngine() const {
+ auto engine = std::make_shared<T>();
+
+ engine->bind("log", logger_);
+ engine->bind("REL_SUCCESS", Success);
+ engine->bind("REL_FAILURE", Failure);
+
+ return std::move(engine);
+ }
+
+ template<typename T>
+ void triggerEngineProcessor(const std::shared_ptr<script::ScriptEngine> &engine,
+ const std::shared_ptr<core::ProcessContext> &context,
+ const std::shared_ptr<core::ProcessSession> &session) const {
+ auto typed_engine = std::static_pointer_cast<T>(engine);
+ typed_engine->call("onTrigger", context, session);
+ }
+};
+
+REGISTER_RESOURCE(ExecuteScript); // NOLINT
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_EXECUTESCRIPT_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ScriptEngine.h
----------------------------------------------------------------------
diff --git a/extensions/script/ScriptEngine.h b/extensions/script/ScriptEngine.h
new file mode 100644
index 0000000..5a41414
--- /dev/null
+++ b/extensions/script/ScriptEngine.h
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_SCRIPTENGINE_H
+#define NIFI_MINIFI_CPP_SCRIPTENGINE_H
+
+#include <string>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace script {
+
+class ScriptEngine {
+ public:
+ /**
+ * Evaluates the given script string, storing the expression's result into res_var, if res_var is not empty.
+ * @param script
+ * @param res_var
+ */
+ virtual void eval(const std::string &script) = 0;
+
+ /**
+ * Evaluates the given script file, storing the expression's result into res_var, if res_var is not empty.
+ * @param script
+ * @param res_var
+ */
+ virtual void evalFile(const std::string &file_name) = 0;
+};
+
+} /* namespace script */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_SCRIPTENGINE_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ScriptFlowFile.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/ScriptFlowFile.cpp b/extensions/script/ScriptFlowFile.cpp
new file mode 100644
index 0000000..79ec073
--- /dev/null
+++ b/extensions/script/ScriptFlowFile.cpp
@@ -0,0 +1,64 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <utility>
+#include <memory>
+#include <string>
+
+#include <core/FlowFile.h>
+
+#include "ScriptFlowFile.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace script {
+
+ScriptFlowFile::ScriptFlowFile(std::shared_ptr<core::FlowFile> flow_file)
+ : flow_file_(std::move(flow_file)) {
+}
+
+std::string ScriptFlowFile::getAttribute(const std::string &key) {
+ std::string value;
+ flow_file_->getAttribute(key, value);
+ return value;
+}
+
+bool ScriptFlowFile::addAttribute(const std::string &key, const std::string &value) {
+ return flow_file_->addAttribute(key, value);
+}
+
+bool ScriptFlowFile::updateAttribute(std::string key, std::string value) {
+ return flow_file_->updateAttribute(std::move(key), std::move(value));
+}
+
+bool ScriptFlowFile::removeAttribute(std::string key) {
+ return flow_file_->removeAttribute(std::move(key));
+}
+
+std::shared_ptr<core::FlowFile> ScriptFlowFile::getFlowFile() {
+ return flow_file_;
+}
+
+} /* namespace script */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ScriptFlowFile.h
----------------------------------------------------------------------
diff --git a/extensions/script/ScriptFlowFile.h b/extensions/script/ScriptFlowFile.h
new file mode 100644
index 0000000..707cf01
--- /dev/null
+++ b/extensions/script/ScriptFlowFile.h
@@ -0,0 +1,49 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_SCRIPTFLOWFILE_H
+#define NIFI_MINIFI_CPP_SCRIPTFLOWFILE_H
+
+#include <core/FlowFile.h>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace script {
+
+class ScriptFlowFile {
+ public:
+ explicit ScriptFlowFile(std::shared_ptr<core::FlowFile> flow_file);
+ std::string getAttribute(const std::string &key);
+ bool addAttribute(const std::string &key, const std::string &value);
+ bool updateAttribute(std::string key, std::string value);
+ bool removeAttribute(std::string key);
+ std::shared_ptr<core::FlowFile> getFlowFile();
+
+ private:
+ std::shared_ptr<core::FlowFile> flow_file_;
+};
+
+} /* namespace script */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_SCRIPTFLOWFILE_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ScriptProcessContext.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/ScriptProcessContext.cpp b/extensions/script/ScriptProcessContext.cpp
new file mode 100644
index 0000000..76aa6b7
--- /dev/null
+++ b/extensions/script/ScriptProcessContext.cpp
@@ -0,0 +1,44 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <string>
+
+#include "ScriptProcessContext.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace script {
+
+ScriptProcessContext::ScriptProcessContext(std::shared_ptr<core::ProcessContext> context)
+ : context_(std::move(context)) {
+}
+
+std::string ScriptProcessContext::getProperty(const std::string &name) {
+ std::string value;
+ context_->getProperty(name, value);
+ return value;
+}
+
+} /* namespace script */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/ScriptProcessContext.h
----------------------------------------------------------------------
diff --git a/extensions/script/ScriptProcessContext.h b/extensions/script/ScriptProcessContext.h
new file mode 100644
index 0000000..4f240b1
--- /dev/null
+++ b/extensions/script/ScriptProcessContext.h
@@ -0,0 +1,48 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_PYPROCESSCONTEXT_H
+#define NIFI_MINIFI_CPP_PYPROCESSCONTEXT_H
+
+#include <core/ProcessSession.h>
+
+#include "ScriptFlowFile.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace script {
+
+class ScriptProcessContext {
+ public:
+ explicit ScriptProcessContext(std::shared_ptr<core::ProcessContext> context);
+
+ std::string getProperty(const std::string &name);
+
+ private:
+ std::shared_ptr<core::ProcessContext> context_;
+};
+
+} /* namespace script */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_PYPROCESSCONTEXT_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/lua/LuaBaseStream.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/lua/LuaBaseStream.cpp b/extensions/script/lua/LuaBaseStream.cpp
new file mode 100644
index 0000000..2f97a22
--- /dev/null
+++ b/extensions/script/lua/LuaBaseStream.cpp
@@ -0,0 +1,73 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdexcept>
+#include <memory>
+#include <utility>
+#include <string>
+
+#include "LuaBaseStream.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace lua {
+
+LuaBaseStream::LuaBaseStream(std::shared_ptr<io::BaseStream> stream)
+ : stream_(std::move(stream)) {
+}
+
+std::string LuaBaseStream::read(size_t len) {
+ if (len == 0) {
+ len = stream_->getSize();
+ }
+
+ if (len <= 0) {
+ return nullptr;
+ }
+
+ std::string buffer;
+ buffer.resize(len);
+
+ // Here, we write directly to the string data, which is safe starting in C++ 11:
+ //
+ // The char-like objects in a basic_string object shall be stored contiguously. That is, for any basic_string
+ // object s, the identity &*(s.begin() + n) == &*s.begin() + n shall hold for all values of n such that
+ // 0 <= n < s.size()."
+ //
+ // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3337.pdf
+ auto read = stream_->readData(reinterpret_cast<uint8_t *>(&buffer[0]), static_cast<int>(len));
+
+ if (read != len) {
+ buffer.resize(static_cast<size_t >(read));
+ }
+
+ return std::move(buffer);
+}
+
+size_t LuaBaseStream::write(std::string buf) {
+ return static_cast<size_t>(stream_->writeData(reinterpret_cast<uint8_t *>(const_cast<char *>(buf.data())),
+ static_cast<int>(buf.length())));
+}
+
+} /* namespace lua */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/lua/LuaBaseStream.h
----------------------------------------------------------------------
diff --git a/extensions/script/lua/LuaBaseStream.h b/extensions/script/lua/LuaBaseStream.h
new file mode 100644
index 0000000..fb19e08
--- /dev/null
+++ b/extensions/script/lua/LuaBaseStream.h
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_LUABASESTREAM_H
+#define NIFI_MINIFI_CPP_LUABASESTREAM_H
+
+#include <memory>
+#include <sol.hpp>
+#include <io/BaseStream.h>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace lua {
+
+class LuaBaseStream {
+ public:
+ explicit LuaBaseStream(std::shared_ptr<io::BaseStream> stream);
+
+ /**
+ * Read n bytes of data (returns string, to follow Lua idioms)
+ * @return
+ */
+ std::string read(size_t len = 0);
+
+ /**
+ * Write data (receives string, to follow Lua idioms)
+ * @param buf
+ * @return
+ */
+ size_t write(std::string buf);
+
+ private:
+ std::shared_ptr<io::BaseStream> stream_;
+};
+
+} /* namespace lua */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_LUABASESTREAM_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/lua/LuaProcessSession.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/lua/LuaProcessSession.cpp b/extensions/script/lua/LuaProcessSession.cpp
new file mode 100644
index 0000000..ce91c62
--- /dev/null
+++ b/extensions/script/lua/LuaProcessSession.cpp
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+
+#include "LuaProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace lua {
+
+LuaProcessSession::LuaProcessSession(std::shared_ptr<core::ProcessSession> session)
+ : session_(std::move(session)) {
+}
+
+std::shared_ptr<script::ScriptFlowFile> LuaProcessSession::get() {
+ auto flow_file = session_->get();
+
+ if (flow_file == nullptr) {
+ return nullptr;
+ }
+
+ return std::make_shared<script::ScriptFlowFile>(flow_file);
+}
+
+void LuaProcessSession::transfer(const std::shared_ptr<script::ScriptFlowFile> &flow_file,
+ core::Relationship relationship) {
+ session_->transfer(flow_file->getFlowFile(), relationship);
+}
+
+void LuaProcessSession::read(const std::shared_ptr<script::ScriptFlowFile> &flow_file,
+ sol::table input_stream_callback) {
+ LuaInputStreamCallback lua_callback(input_stream_callback);
+ session_->read(flow_file->getFlowFile(), &lua_callback);
+}
+
+void LuaProcessSession::write(const std::shared_ptr<script::ScriptFlowFile> &flow_file,
+ sol::table output_stream_callback) {
+ LuaOutputStreamCallback lua_callback(output_stream_callback);
+ session_->write(flow_file->getFlowFile(), &lua_callback);
+}
+
+std::shared_ptr<script::ScriptFlowFile> LuaProcessSession::create() {
+ return std::make_shared<script::ScriptFlowFile>(session_->create());
+}
+
+std::shared_ptr<script::ScriptFlowFile> LuaProcessSession::create(const std::shared_ptr<script::ScriptFlowFile> &flow_file) {
+ if (flow_file == nullptr) {
+ return std::make_shared<script::ScriptFlowFile>(session_->create());
+ } else {
+ return std::make_shared<script::ScriptFlowFile>(session_->create(flow_file->getFlowFile()));
+ }
+}
+
+} /* namespace lua */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/lua/LuaProcessSession.h
----------------------------------------------------------------------
diff --git a/extensions/script/lua/LuaProcessSession.h b/extensions/script/lua/LuaProcessSession.h
new file mode 100644
index 0000000..4752d0d
--- /dev/null
+++ b/extensions/script/lua/LuaProcessSession.h
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_LUAPROCESSSESSION_H
+#define NIFI_MINIFI_CPP_LUAPROCESSSESSION_H
+
+#include <sol.hpp>
+
+#include <core/ProcessSession.h>
+
+#include "../ScriptFlowFile.h"
+
+#include "LuaBaseStream.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace lua {
+
+class LuaProcessSession {
+ public:
+ explicit LuaProcessSession(std::shared_ptr<core::ProcessSession> session);
+
+ std::shared_ptr<script::ScriptFlowFile> get();
+ std::shared_ptr<script::ScriptFlowFile> create();
+ std::shared_ptr<script::ScriptFlowFile> create(const std::shared_ptr<script::ScriptFlowFile> &flow_file);
+ void transfer(const std::shared_ptr<script::ScriptFlowFile> &flow_file, core::Relationship relationship);
+ void read(const std::shared_ptr<script::ScriptFlowFile> &flow_file, sol::table input_stream_callback);
+ void write(const std::shared_ptr<script::ScriptFlowFile> &flow_file, sol::table output_stream_callback);
+
+ class LuaInputStreamCallback : public InputStreamCallback {
+ public:
+ explicit LuaInputStreamCallback(const sol::table &input_stream_callback) {
+ lua_callback_ = input_stream_callback;
+ }
+
+ int64_t process(std::shared_ptr<io::BaseStream> stream) override {
+ auto lua_stream = std::make_shared<LuaBaseStream>(stream);
+ sol::function callback = lua_callback_["process"];
+ return callback(lua_callback_, lua_stream);
+ }
+
+ private:
+ sol::table lua_callback_;
+ };
+
+ class LuaOutputStreamCallback : public OutputStreamCallback {
+ public:
+ explicit LuaOutputStreamCallback(const sol::table &output_stream_callback) {
+ lua_callback_ = output_stream_callback;
+ }
+
+ int64_t process(std::shared_ptr<io::BaseStream> stream) override {
+ auto lua_stream = std::make_shared<LuaBaseStream>(stream);
+ sol::function callback = lua_callback_["process"];
+ return callback(lua_callback_, lua_stream);
+ }
+
+ private:
+ sol::table lua_callback_;
+ };
+
+ private:
+ std::shared_ptr<core::ProcessSession> session_;
+};
+
+} /* namespace lua */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_LUAPROCESSSESSION_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/lua/LuaScriptEngine.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/lua/LuaScriptEngine.cpp b/extensions/script/lua/LuaScriptEngine.cpp
new file mode 100644
index 0000000..718014b
--- /dev/null
+++ b/extensions/script/lua/LuaScriptEngine.cpp
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <string>
+
+#include "../ScriptProcessContext.h"
+
+#include "LuaScriptEngine.h"
+#include "LuaProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace lua {
+
+LuaScriptEngine::LuaScriptEngine()
+ : lua_() {
+ lua_.new_usertype<core::logging::Logger>(
+ "Logger",
+ "info", &core::logging::Logger::log_info<>);
+ lua_.new_usertype<lua::LuaProcessSession>(
+ "ProcessSession",
+ "create", static_cast<std::shared_ptr<script::ScriptFlowFile> (lua::LuaProcessSession::*)()>(&lua::LuaProcessSession::create),
+ "get", &lua::LuaProcessSession::get,
+ "read", &lua::LuaProcessSession::read,
+ "write", &lua::LuaProcessSession::write,
+ "transfer", &lua::LuaProcessSession::transfer);
+ lua_.new_usertype<script::ScriptFlowFile>(
+ "ProcessSession",
+ "getAttribute", &script::ScriptFlowFile::getAttribute,
+ "addAttribute", &script::ScriptFlowFile::addAttribute,
+ "removeAttribute", &script::ScriptFlowFile::removeAttribute,
+ "updateAttribute", &script::ScriptFlowFile::updateAttribute);
+ lua_.new_usertype<lua::LuaBaseStream>(
+ "BaseStream",
+ "read", &lua::LuaBaseStream::read,
+ "write", &lua::LuaBaseStream::write);
+}
+
+void LuaScriptEngine::eval(const std::string &script) {
+ lua_.script(script);
+}
+
+void LuaScriptEngine::evalFile(const std::string &file_name) {
+ lua_.script_file(file_name);
+}
+
+} /* namespace lua */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/lua/LuaScriptEngine.h
----------------------------------------------------------------------
diff --git a/extensions/script/lua/LuaScriptEngine.h b/extensions/script/lua/LuaScriptEngine.h
new file mode 100644
index 0000000..ec40ddb
--- /dev/null
+++ b/extensions/script/lua/LuaScriptEngine.h
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_LUASCRIPTENGINE_H
+#define NIFI_MINIFI_CPP_LUASCRIPTENGINE_H
+
+#include <mutex>
+#include <sol.hpp>
+#include <core/ProcessSession.h>
+
+#include "../ScriptEngine.h"
+#include "../ScriptProcessContext.h"
+
+#include "LuaProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace lua {
+
+class LuaScriptEngine : public script::ScriptEngine {
+ public:
+ LuaScriptEngine();
+
+ void eval(const std::string &script) override;
+ void evalFile(const std::string &file_name) override;
+
+ /**
+ * Calls the given function, forwarding arbitrary provided parameters.
+ *
+ * @return
+ */
+ template<typename... Args>
+ void call(const std::string &fn_name, Args &&...args) {
+ sol::function fn = lua_[fn_name.c_str()];
+ fn(convert(args)...);
+ }
+
+
+ template<typename T>
+ void bind(const std::string &name, const T &value) {
+ lua_[name.c_str()] = convert(value);
+ }
+
+ template<typename T>
+ T convert(const T &value) {
+ return value;
+ }
+
+ std::shared_ptr<LuaProcessSession> convert(const std::shared_ptr<core::ProcessSession> &session) {
+ return std::make_shared<LuaProcessSession>(session);
+ }
+
+ std::shared_ptr<script::ScriptProcessContext> convert(const std::shared_ptr<core::ProcessContext> &context) {
+ return std::make_shared<script::ScriptProcessContext>(context);
+ }
+
+ private:
+ sol::state lua_;
+};
+
+} /* namespace lua */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_LUASCRIPTENGINE_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PyBaseStream.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/python/PyBaseStream.cpp b/extensions/script/python/PyBaseStream.cpp
new file mode 100644
index 0000000..3e32e5b
--- /dev/null
+++ b/extensions/script/python/PyBaseStream.cpp
@@ -0,0 +1,68 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdexcept>
+#include <memory>
+#include <utility>
+#include <string>
+#include <vector>
+
+#include "PyBaseStream.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace python {
+
+PyBaseStream::PyBaseStream(std::shared_ptr<io::BaseStream> stream)
+ : stream_(std::move(stream)) {
+}
+
+py::bytes PyBaseStream::read() {
+ return read(stream_->getSize());
+}
+
+py::bytes PyBaseStream::read(size_t len) {
+ if (len == 0) {
+ len = stream_->getSize();
+ }
+
+ if (len <= 0) {
+ return nullptr;
+ }
+
+ std::vector<uint8_t> buffer(len);
+
+ auto read = stream_->readData(buffer.data(), static_cast<int>(len));
+ auto result = py::bytes(reinterpret_cast<char *>(buffer.data()), static_cast<size_t>(read));
+
+ return std::move(result);
+}
+
+size_t PyBaseStream::write(py::bytes buf) {
+ const auto &&buf_str = buf.operator std::string();
+ return static_cast<size_t>(stream_->writeData(reinterpret_cast<uint8_t *>(const_cast<char *>(buf_str.data())),
+ static_cast<int>(buf_str.length())));
+}
+
+} /* namespace python */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PyBaseStream.h
----------------------------------------------------------------------
diff --git a/extensions/script/python/PyBaseStream.h b/extensions/script/python/PyBaseStream.h
new file mode 100644
index 0000000..475d50a
--- /dev/null
+++ b/extensions/script/python/PyBaseStream.h
@@ -0,0 +1,52 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_PYBYTESTREAM_H
+#define NIFI_MINIFI_CPP_PYBYTESTREAM_H
+
+#include <pybind11/embed.h>
+
+#include <io/BaseStream.h>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace python {
+
+namespace py = pybind11;
+
+class PyBaseStream {
+ public:
+ explicit PyBaseStream(std::shared_ptr<io::BaseStream> stream);
+
+ py::bytes read();
+ py::bytes read(size_t len = 0);
+ size_t write(py::bytes buf);
+
+ private:
+ std::shared_ptr<io::BaseStream> stream_;
+};
+
+} /* namespace python */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_PYBYTESTREAM_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PyProcessSession.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/python/PyProcessSession.cpp b/extensions/script/python/PyProcessSession.cpp
new file mode 100644
index 0000000..2a72ad2
--- /dev/null
+++ b/extensions/script/python/PyProcessSession.cpp
@@ -0,0 +1,77 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+
+#include <pybind11/embed.h>
+
+#include "PyProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace python {
+
+namespace py = pybind11;
+namespace core = org::apache::nifi::minifi::core;
+
+PyProcessSession::PyProcessSession(std::shared_ptr<core::ProcessSession> session)
+ : session_(std::move(session)) {
+}
+
+std::shared_ptr<script::ScriptFlowFile> PyProcessSession::get() {
+ auto flow_file = session_->get();
+
+ if (flow_file == nullptr) {
+ return nullptr;
+ }
+
+ return std::make_shared<script::ScriptFlowFile>(flow_file);
+}
+
+void PyProcessSession::transfer(std::shared_ptr<script::ScriptFlowFile> flow_file,
+ core::Relationship relationship) {
+ session_->transfer(flow_file->getFlowFile(), relationship);
+}
+
+void PyProcessSession::read(std::shared_ptr<script::ScriptFlowFile> flow_file,
+ py::object input_stream_callback) {
+ PyInputStreamCallback py_callback(input_stream_callback);
+ session_->read(flow_file->getFlowFile(), &py_callback);
+}
+
+void PyProcessSession::write(std::shared_ptr<script::ScriptFlowFile> flow_file,
+ py::object output_stream_callback) {
+ PyOutputStreamCallback py_callback(output_stream_callback);
+ session_->write(flow_file->getFlowFile(), &py_callback);
+}
+
+std::shared_ptr<script::ScriptFlowFile> PyProcessSession::create() {
+ return std::make_shared<script::ScriptFlowFile>(session_->create());
+}
+
+std::shared_ptr<script::ScriptFlowFile> PyProcessSession::create(std::shared_ptr<script::ScriptFlowFile> flow_file) {
+ return std::make_shared<script::ScriptFlowFile>(session_->create(flow_file->getFlowFile()));
+}
+
+} /* namespace python */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PyProcessSession.h
----------------------------------------------------------------------
diff --git a/extensions/script/python/PyProcessSession.h b/extensions/script/python/PyProcessSession.h
new file mode 100644
index 0000000..3581866
--- /dev/null
+++ b/extensions/script/python/PyProcessSession.h
@@ -0,0 +1,89 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_PYPROCESSESSION_H
+#define NIFI_MINIFI_CPP_PYPROCESSESSION_H
+
+#include <pybind11/embed.h>
+
+#include <core/ProcessSession.h>
+
+#include "../ScriptFlowFile.h"
+
+#include "PyBaseStream.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace python {
+
+namespace py = pybind11;
+
+class PyProcessSession {
+ public:
+ explicit PyProcessSession(std::shared_ptr<core::ProcessSession> session);
+
+ std::shared_ptr<script::ScriptFlowFile> get();
+ std::shared_ptr<script::ScriptFlowFile> create();
+ std::shared_ptr<script::ScriptFlowFile> create(std::shared_ptr<script::ScriptFlowFile> flow_file);
+ void transfer(std::shared_ptr<script::ScriptFlowFile> flow_file, core::Relationship relationship);
+ void read(std::shared_ptr<script::ScriptFlowFile> flow_file, py::object input_stream_callback);
+ void write(std::shared_ptr<script::ScriptFlowFile> flow_file, py::object output_stream_callback);
+
+ class __attribute__((visibility("default"))) PyInputStreamCallback : public InputStreamCallback {
+ public:
+ explicit PyInputStreamCallback(const py::object &input_stream_callback) {
+ py_callback_ = input_stream_callback;
+ }
+
+ int64_t process(std::shared_ptr<io::BaseStream> stream) override {
+ auto py_stream = std::make_shared<PyBaseStream>(stream);
+ return py_callback_.attr("process")(py_stream).cast<int64_t>();
+ }
+
+ private:
+ py::object py_callback_;
+ };
+
+ class __attribute__((visibility("default"))) PyOutputStreamCallback : public OutputStreamCallback {
+ public:
+ explicit PyOutputStreamCallback(const py::object &output_stream_callback) {
+ py_callback_ = output_stream_callback;
+ }
+
+ int64_t process(std::shared_ptr<io::BaseStream> stream) override {
+ auto py_stream = std::make_shared<PyBaseStream>(stream);
+ return py_callback_.attr("process")(py_stream).cast<int64_t>();
+ }
+
+ private:
+ py::object py_callback_;
+ };
+
+ private:
+ std::shared_ptr<core::ProcessSession> session_;
+};
+
+} /* namespace python */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_PYPROCESSESSION_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PythonBindings.h
----------------------------------------------------------------------
diff --git a/extensions/script/python/PythonBindings.h b/extensions/script/python/PythonBindings.h
new file mode 100644
index 0000000..9a482ef
--- /dev/null
+++ b/extensions/script/python/PythonBindings.h
@@ -0,0 +1,77 @@
+/**
+ * @file PythonBindings.h
+
+ * Python C++ type bindings
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_PYTHONBINDINGS_H
+#define NIFI_MINIFI_CPP_PYTHONBINDINGS_H
+
+#include <pybind11/embed.h>
+
+#include <core/ProcessSession.h>
+#include <core/logging/LoggerConfiguration.h>
+
+#include "../ScriptProcessContext.h"
+
+#include "PyProcessSession.h"
+#include "PyBaseStream.h"
+
+PYBIND11_EMBEDDED_MODULE(minifi_native, m) { // NOLINT
+ namespace py = pybind11;
+ namespace python = org::apache::nifi::minifi::python;
+ namespace script = org::apache::nifi::minifi::script;
+ typedef core::logging::Logger Logger;
+
+ py::class_<Logger, std::shared_ptr<Logger>>(m, "Logger")
+ .def("error", &Logger::log_error<>)
+ .def("warn", &Logger::log_warn<>)
+ .def("info", &Logger::log_info<>)
+ .def("debug", &Logger::log_debug<>)
+ .def("trace", &Logger::log_trace<>);
+
+ py::class_<python::PyProcessSession, std::shared_ptr<python::PyProcessSession>>(m, "ProcessSession")
+ .def("get", &python::PyProcessSession::get, py::return_value_policy::reference)
+ .def("create",
+ static_cast<std::shared_ptr<script::ScriptFlowFile> (python::PyProcessSession::*)()>(&python::PyProcessSession::create))
+ .def("create",
+ static_cast<std::shared_ptr<script::ScriptFlowFile> (python::PyProcessSession::*)(std::shared_ptr<script::ScriptFlowFile>)>(&python::PyProcessSession::create))
+ .def("read", &python::PyProcessSession::read)
+ .def("write", &python::PyProcessSession::write)
+ .def("transfer", &python::PyProcessSession::transfer);
+
+ py::class_<script::ScriptProcessContext, std::shared_ptr<script::ScriptProcessContext>>(m, "ProcessContext")
+ .def("getProperty", &script::ScriptProcessContext::getProperty);
+
+ py::class_<script::ScriptFlowFile, std::shared_ptr<script::ScriptFlowFile>>(m, "FlowFile")
+ .def("getAttribute", &script::ScriptFlowFile::getAttribute)
+ .def("addAttribute", &script::ScriptFlowFile::addAttribute)
+ .def("updateAttribute", &script::ScriptFlowFile::updateAttribute)
+ .def("removeAttribute", &script::ScriptFlowFile::removeAttribute);
+
+ py::class_<core::Relationship>(m, "Relationship")
+ .def("getName", &core::Relationship::getName)
+ .def("getDescription", &core::Relationship::getDescription);
+
+ py::class_<python::PyBaseStream, std::shared_ptr<python::PyBaseStream>>(m, "BaseStream")
+ .def("read", static_cast<py::bytes (python::PyBaseStream::*)()>(&python::PyBaseStream::read))
+ .def("read", static_cast<py::bytes (python::PyBaseStream::*)(size_t)>(&python::PyBaseStream::read))
+ .def("write", &python::PyBaseStream::write);
+}
+
+#endif //NIFI_MINIFI_CPP_PYTHONBINDINGS_H
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PythonScriptEngine.cpp
----------------------------------------------------------------------
diff --git a/extensions/script/python/PythonScriptEngine.cpp b/extensions/script/python/PythonScriptEngine.cpp
new file mode 100644
index 0000000..04bc88f
--- /dev/null
+++ b/extensions/script/python/PythonScriptEngine.cpp
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <string>
+
+#include "PythonBindings.h"
+#include "PythonScriptEngine.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace python {
+
+std::unique_ptr<py::scoped_interpreter> PythonScriptEngine::guard_;
+std::unique_ptr<py::gil_scoped_release> PythonScriptEngine::gil_release_;
+std::mutex PythonScriptEngine::init_mutex_;
+bool PythonScriptEngine::initialized_;
+
+PythonScriptEngine::PythonScriptEngine() {
+ {
+ std::lock_guard<std::mutex> lock(init_mutex_);
+
+ if (!initialized_) {
+ throw std::runtime_error("Python is not yet initialized");
+ }
+ }
+
+ {
+ py::gil_scoped_acquire gil{};
+ py::module::import("minifi_native");
+ bindings_.reset(new py::dict());
+ (*bindings_) = py::globals().attr("copy")();
+ }
+}
+
+void PythonScriptEngine::eval(const std::string &script) {
+ py::gil_scoped_acquire gil{};
+
+ if (script[0] == '\n') {
+ py::eval<py::eval_statements>(py::module::import("textwrap").attr("dedent")(script), *bindings_, *bindings_);
+ } else {
+ py::eval<py::eval_statements>(script, *bindings_, *bindings_);
+ }
+}
+
+void PythonScriptEngine::evalFile(const std::string &file_name) {
+ py::gil_scoped_acquire gil{};
+ py::eval_file(file_name, *bindings_, *bindings_);
+}
+
+void PythonScriptEngine::initialize() {
+ std::lock_guard<std::mutex> lock(init_mutex_);
+
+ if (initialized_) {
+ return;
+ }
+
+ initialized_ = true;
+ guard_.reset(new py::scoped_interpreter());
+ gil_release_.reset(new py::gil_scoped_release());
+}
+
+} /* namespace python */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/extensions/script/python/PythonScriptEngine.h
----------------------------------------------------------------------
diff --git a/extensions/script/python/PythonScriptEngine.h b/extensions/script/python/PythonScriptEngine.h
new file mode 100644
index 0000000..575b4db
--- /dev/null
+++ b/extensions/script/python/PythonScriptEngine.h
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NIFI_MINIFI_CPP_PYTHONSCRIPTENGINE_H
+#define NIFI_MINIFI_CPP_PYTHONSCRIPTENGINE_H
+
+#include <mutex>
+#include <pybind11/embed.h>
+#include <core/ProcessSession.h>
+
+#include "../ScriptEngine.h"
+#include "../ScriptProcessContext.h"
+
+#include "PyProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace python {
+
+namespace py = pybind11;
+
+class __attribute__((visibility("default"))) PythonScriptEngine : public script::ScriptEngine {
+ public:
+ PythonScriptEngine();
+ ~PythonScriptEngine() {
+ py::gil_scoped_acquire gil{};
+ (*bindings_).dec_ref();
+ (*bindings_).release();
+ }
+
+ /**
+ * Initializes the python interpreter.
+ */
+ static void initialize();
+
+ /**
+ * Evaluates a python expression.
+ *
+ * @param script
+ */
+ void eval(const std::string &script) override;
+
+ /**
+ * Evaluates the contents of the given python file name.
+ *
+ * @param file_name
+ */
+ void evalFile(const std::string &file_name) override;
+
+ /**
+ * Calls the given function, forwarding arbitrary provided parameters.
+ *
+ * @return
+ */
+ template<typename... Args>
+ void call(const std::string &fn_name, Args &&...args) {
+ py::gil_scoped_acquire gil{};
+ (*bindings_)[fn_name.c_str()](convert(args)...);
+ }
+
+ /**
+ * Binds an object into the scope of the python interpreter.
+ * @tparam T
+ * @param name
+ * @param value
+ */
+ template<typename T>
+ void bind(const std::string &name, const T &value) {
+ py::gil_scoped_acquire gil{};
+ (*bindings_)[name.c_str()] = convert(value);
+ }
+
+ template<typename T>
+ py::object convert(const T &value) {
+ py::gil_scoped_acquire gil{};
+ return py::cast(value);
+ }
+
+ py::object convert(const std::shared_ptr<core::ProcessSession> &session) {
+ return py::cast(std::make_shared<PyProcessSession>(session));
+ }
+
+ py::object convert(const std::shared_ptr<core::ProcessContext> &context) {
+ return py::cast(std::make_shared<script::ScriptProcessContext>(context));
+ }
+
+ private:
+ static std::unique_ptr<py::scoped_interpreter> guard_;
+ static std::unique_ptr<py::gil_scoped_release> gil_release_;
+
+ static std::mutex init_mutex_;
+ static bool initialized_;
+ std::unique_ptr<py::dict> bindings_;
+};
+
+} /* namespace python */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif //NIFI_MINIFI_CPP_PYTHONSCRIPTENGINE_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/9a10b98e/libminifi/test/script-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libminifi/test/script-tests/CMakeLists.txt b/libminifi/test/script-tests/CMakeLists.txt
new file mode 100644
index 0000000..f0a57d0
--- /dev/null
+++ b/libminifi/test/script-tests/CMakeLists.txt
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if (NOT DISABLE_PYTHON_SCRIPTING)
+ file(GLOB EXECUTESCRIPT_PYTHON_INTEGRATION_TESTS "Python*.cpp")
+endif()
+
+if (ENABLE_LUA_SCRIPTING)
+ file(GLOB EXECUTESCRIPT_LUA_INTEGRATION_TESTS "Lua*.cpp")
+endif()
+
+SET(EXTENSIONS_TEST_COUNT 0)
+
+FOREACH(testfile ${EXECUTESCRIPT_PYTHON_INTEGRATION_TESTS})
+ get_filename_component(testfilename "${testfile}" NAME_WE)
+ add_executable("${testfilename}" "${testfile}" ${SPD_SOURCES} "${TEST_DIR}/TestBase.cpp")
+ createTests("${testfilename}")
+ target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script")
+ target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script/python")
+ target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/thirdparty/pybind11/include")
+ add_definitions(-DPYTHON_SUPPORT)
+ if (APPLE)
+ target_link_libraries ("${testfilename}" -Wl,-all_load minifi-script-extensions )
+ else ()
+ target_link_libraries ("${testfilename}" -Wl,--whole-archive minifi-script-extensions -Wl,--no-whole-archive)
+ endif()
+ MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1")
+ add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
+ENDFOREACH()
+
+FOREACH(testfile ${EXECUTESCRIPT_LUA_INTEGRATION_TESTS})
+ get_filename_component(testfilename "${testfile}" NAME_WE)
+ add_executable("${testfilename}" "${testfile}" ${SPD_SOURCES} "${TEST_DIR}/TestBase.cpp")
+ createTests("${testfilename}")
+ target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script")
+ target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script/lua")
+ target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/thirdparty/sol2-2.17.5")
+ add_definitions(-DLUA_SUPPORT)
+ if (APPLE)
+ target_link_libraries ("${testfilename}" -Wl,-all_load minifi-script-extensions )
+ else ()
+ target_link_libraries ("${testfilename}" -Wl,--whole-archive minifi-script-extensions -Wl,--no-whole-archive)
+ endif()
+ MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1")
+ add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
+ENDFOREACH()
+
+message("-- Finished building ${PYTHON_EXECUTESCRIPT-EXTENSIONS_TEST_COUNT} Python ExecuteScript related test file(s)...")