You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2017/10/26 01:21:16 UTC
[13/13] nifi-minifi-cpp git commit: MINIFICPP-263: Move merge content
to an extension. Also remove circular dependencies
MINIFICPP-263: Move merge content to an extension. Also remove circular dependencies
MINIFICPP-263: Remove boost deps by updating yaml-cpp
MINIFICPP-263: Resolve issues on u14
This closes #153.
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/commit/1d6b2416
Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/tree/1d6b2416
Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/diff/1d6b2416
Branch: refs/heads/master
Commit: 1d6b2416703fbb706d7936e4052ea9bc3da0a08b
Parents: dd10240
Author: Marc Parisi <ph...@apache.org>
Authored: Mon Oct 23 21:04:00 2017 -0400
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Oct 25 21:19:32 2017 -0400
----------------------------------------------------------------------
CMakeLists.txt | 20 +-
cmake/BuildTests.cmake | 25 +-
extensions/http-curl/CMakeLists.txt | 6 +-
extensions/libarchive/ArchiveLoader.cpp | 29 +
extensions/libarchive/ArchiveLoader.h | 67 ++
extensions/libarchive/BinFiles.cpp | 304 +++++
extensions/libarchive/BinFiles.h | 308 +++++
extensions/libarchive/CMakeLists.txt | 75 ++
extensions/libarchive/MergeContent.cpp | 334 ++++++
extensions/libarchive/MergeContent.h | 342 ++++++
extensions/rocksdb-repos/CMakeLists.txt | 10 +-
.../rocksdb-repos/DatabaseContentRepository.h | 2 +-
libminifi/CMakeLists.txt | 48 +-
libminifi/include/core/FlowConfiguration.h | 1 -
libminifi/include/processors/BinFiles.h | 308 -----
libminifi/include/processors/LoadProcessors.h | 1 -
libminifi/include/processors/MergeContent.h | 342 ------
libminifi/src/core/ConfigurationFactory.cpp | 5 -
libminifi/src/processors/BinFiles.cpp | 304 -----
libminifi/src/processors/MergeContent.cpp | 334 ------
libminifi/test/archive-tests/MergeFileTests.cpp | 1061 ++++++++++++++++++
libminifi/test/unit/MergeFileTests.cpp | 1061 ------------------
main/CMakeLists.txt | 13 +-
.../yaml-cpp-yaml-cpp-0.5.3/.clang-format | 47 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/.gitignore | 1 -
.../yaml-cpp-yaml-cpp-0.5.3/CMakeLists.txt | 337 ------
.../yaml-cpp-yaml-cpp-0.5.3/CONTRIBUTING.md | 17 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/LICENSE | 19 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/README.md | 52 -
.../include/yaml-cpp/anchor.h | 17 -
.../include/yaml-cpp/binary.h | 67 --
.../include/yaml-cpp/contrib/anchordict.h | 37 -
.../include/yaml-cpp/contrib/graphbuilder.h | 147 ---
.../include/yaml-cpp/dll.h | 37 -
.../include/yaml-cpp/emitfromevents.h | 57 -
.../include/yaml-cpp/emitter.h | 254 -----
.../include/yaml-cpp/emitterdef.h | 16 -
.../include/yaml-cpp/emittermanip.h | 137 ---
.../include/yaml-cpp/emitterstyle.h | 16 -
.../include/yaml-cpp/eventhandler.h | 40 -
.../include/yaml-cpp/exceptions.h | 231 ----
.../include/yaml-cpp/mark.h | 29 -
.../include/yaml-cpp/node/convert.h | 297 -----
.../include/yaml-cpp/node/detail/bool_type.h | 26 -
.../include/yaml-cpp/node/detail/impl.h | 177 ---
.../include/yaml-cpp/node/detail/iterator.h | 65 --
.../include/yaml-cpp/node/detail/iterator_fwd.h | 28 -
.../include/yaml-cpp/node/detail/memory.h | 46 -
.../include/yaml-cpp/node/detail/node.h | 170 ---
.../include/yaml-cpp/node/detail/node_data.h | 127 ---
.../yaml-cpp/node/detail/node_iterator.h | 159 ---
.../include/yaml-cpp/node/detail/node_ref.h | 97 --
.../include/yaml-cpp/node/emit.h | 25 -
.../include/yaml-cpp/node/impl.h | 448 --------
.../include/yaml-cpp/node/iterator.h | 31 -
.../include/yaml-cpp/node/node.h | 145 ---
.../include/yaml-cpp/node/parse.h | 30 -
.../include/yaml-cpp/node/ptr.h | 29 -
.../include/yaml-cpp/node/type.h | 16 -
.../include/yaml-cpp/noncopyable.h | 25 -
.../include/yaml-cpp/null.h | 24 -
.../include/yaml-cpp/ostream_wrapper.h | 72 --
.../include/yaml-cpp/parser.h | 48 -
.../include/yaml-cpp/stlemitter.h | 51 -
.../include/yaml-cpp/traits.h | 103 --
.../include/yaml-cpp/yaml.h | 24 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/install.txt | 24 -
.../yaml-cpp-yaml-cpp-0.5.3/src/binary.cpp | 93 --
.../src/collectionstack.h | 39 -
.../src/contrib/graphbuilder.cpp | 17 -
.../src/contrib/graphbuilderadapter.cpp | 94 --
.../src/contrib/graphbuilderadapter.h | 79 --
.../yaml-cpp-yaml-cpp-0.5.3/src/convert.cpp | 75 --
.../yaml-cpp-yaml-cpp-0.5.3/src/directives.cpp | 22 -
.../yaml-cpp-yaml-cpp-0.5.3/src/directives.h | 29 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/emit.cpp | 25 -
.../src/emitfromevents.cpp | 119 --
.../yaml-cpp-yaml-cpp-0.5.3/src/emitter.cpp | 911 ---------------
.../src/emitterstate.cpp | 350 ------
.../yaml-cpp-yaml-cpp-0.5.3/src/emitterstate.h | 203 ----
.../src/emitterutils.cpp | 484 --------
.../yaml-cpp-yaml-cpp-0.5.3/src/emitterutils.h | 50 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/exp.cpp | 136 ---
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/exp.h | 209 ----
.../yaml-cpp-yaml-cpp-0.5.3/src/indentation.h | 41 -
.../yaml-cpp-yaml-cpp-0.5.3/src/memory.cpp | 26 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/node.cpp | 12 -
.../yaml-cpp-yaml-cpp-0.5.3/src/node_data.cpp | 301 -----
.../yaml-cpp-yaml-cpp-0.5.3/src/nodebuilder.cpp | 131 ---
.../yaml-cpp-yaml-cpp-0.5.3/src/nodebuilder.h | 70 --
.../yaml-cpp-yaml-cpp-0.5.3/src/nodeevents.cpp | 101 --
.../yaml-cpp-yaml-cpp-0.5.3/src/nodeevents.h | 64 --
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/null.cpp | 5 -
.../src/ostream_wrapper.cpp | 57 -
.../yaml-cpp-yaml-cpp-0.5.3/src/parse.cpp | 68 --
.../yaml-cpp-yaml-cpp-0.5.3/src/parser.cpp | 128 ---
.../yaml-cpp-yaml-cpp-0.5.3/src/ptr_stack.h | 53 -
.../yaml-cpp-yaml-cpp-0.5.3/src/ptr_vector.h | 49 -
.../yaml-cpp-yaml-cpp-0.5.3/src/regex_yaml.cpp | 45 -
.../yaml-cpp-yaml-cpp-0.5.3/src/regex_yaml.h | 85 --
.../yaml-cpp-yaml-cpp-0.5.3/src/regeximpl.h | 186 ---
.../yaml-cpp-yaml-cpp-0.5.3/src/scanner.cpp | 386 -------
.../yaml-cpp-yaml-cpp-0.5.3/src/scanner.h | 135 ---
.../yaml-cpp-yaml-cpp-0.5.3/src/scanscalar.cpp | 221 ----
.../yaml-cpp-yaml-cpp-0.5.3/src/scanscalar.h | 61 -
.../yaml-cpp-yaml-cpp-0.5.3/src/scantag.cpp | 81 --
.../yaml-cpp-yaml-cpp-0.5.3/src/scantag.h | 19 -
.../yaml-cpp-yaml-cpp-0.5.3/src/scantoken.cpp | 436 -------
.../yaml-cpp-yaml-cpp-0.5.3/src/setting.h | 99 --
.../yaml-cpp-yaml-cpp-0.5.3/src/simplekey.cpp | 128 ---
.../src/singledocparser.cpp | 413 -------
.../src/singledocparser.h | 65 --
.../yaml-cpp-yaml-cpp-0.5.3/src/stream.cpp | 448 --------
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/stream.h | 76 --
.../src/streamcharsource.h | 48 -
.../yaml-cpp-yaml-cpp-0.5.3/src/stringsource.h | 48 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/tag.cpp | 49 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/tag.h | 33 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/src/token.h | 69 --
.../yaml-cpp-yaml-cpp-0.5.3/util/CMakeLists.txt | 11 -
thirdparty/yaml-cpp-yaml-cpp-0.5.3/util/api.cpp | 137 ---
.../yaml-cpp-yaml-cpp-0.5.3/util/parse.cpp | 61 -
.../yaml-cpp-yaml-cpp-0.5.3/util/read.cpp | 33 -
.../yaml-cpp-yaml-cpp-0.5.3/util/sandbox.cpp | 36 -
.../yaml-cpp-config-version.cmake.in | 11 -
.../yaml-cpp-config.cmake.in | 14 -
.../yaml-cpp-yaml-cpp-0.5.3/yaml-cpp.pc.cmake | 11 -
.../yaml-cpp-yaml-cpp-20171024/CMakeLists.txt | 366 ++++++
.../yaml-cpp-yaml-cpp-20171024/CONTRIBUTING.md | 26 +
thirdparty/yaml-cpp-yaml-cpp-20171024/LICENSE | 19 +
thirdparty/yaml-cpp-yaml-cpp-20171024/README.md | 51 +
.../include/yaml-cpp/anchor.h | 17 +
.../include/yaml-cpp/binary.h | 67 ++
.../include/yaml-cpp/contrib/anchordict.h | 39 +
.../include/yaml-cpp/contrib/graphbuilder.h | 147 +++
.../include/yaml-cpp/dll.h | 33 +
.../include/yaml-cpp/emitfromevents.h | 57 +
.../include/yaml-cpp/emitter.h | 254 +++++
.../include/yaml-cpp/emitterdef.h | 16 +
.../include/yaml-cpp/emittermanip.h | 137 +++
.../include/yaml-cpp/emitterstyle.h | 16 +
.../include/yaml-cpp/eventhandler.h | 40 +
.../include/yaml-cpp/exceptions.h | 267 +++++
.../include/yaml-cpp/mark.h | 29 +
.../include/yaml-cpp/node/convert.h | 331 ++++++
.../include/yaml-cpp/node/detail/bool_type.h | 26 +
.../include/yaml-cpp/node/detail/impl.h | 185 +++
.../include/yaml-cpp/node/detail/iterator.h | 91 ++
.../include/yaml-cpp/node/detail/iterator_fwd.h | 27 +
.../include/yaml-cpp/node/detail/memory.h | 46 +
.../include/yaml-cpp/node/detail/node.h | 169 +++
.../include/yaml-cpp/node/detail/node_data.h | 127 +++
.../yaml-cpp/node/detail/node_iterator.h | 180 +++
.../include/yaml-cpp/node/detail/node_ref.h | 98 ++
.../include/yaml-cpp/node/emit.h | 32 +
.../include/yaml-cpp/node/impl.h | 448 ++++++++
.../include/yaml-cpp/node/iterator.h | 31 +
.../include/yaml-cpp/node/node.h | 145 +++
.../include/yaml-cpp/node/parse.h | 78 ++
.../include/yaml-cpp/node/ptr.h | 29 +
.../include/yaml-cpp/node/type.h | 16 +
.../include/yaml-cpp/noncopyable.h | 25 +
.../include/yaml-cpp/null.h | 26 +
.../include/yaml-cpp/ostream_wrapper.h | 72 ++
.../include/yaml-cpp/parser.h | 86 ++
.../include/yaml-cpp/stlemitter.h | 51 +
.../include/yaml-cpp/traits.h | 103 ++
.../include/yaml-cpp/yaml.h | 24 +
.../yaml-cpp-yaml-cpp-20171024/install.txt | 24 +
.../yaml-cpp-yaml-cpp-20171024/src/binary.cpp | 93 ++
.../src/collectionstack.h | 39 +
.../src/contrib/graphbuilder.cpp | 17 +
.../src/contrib/graphbuilderadapter.cpp | 94 ++
.../src/contrib/graphbuilderadapter.h | 79 ++
.../yaml-cpp-yaml-cpp-20171024/src/convert.cpp | 75 ++
.../src/directives.cpp | 22 +
.../yaml-cpp-yaml-cpp-20171024/src/directives.h | 29 +
.../yaml-cpp-yaml-cpp-20171024/src/emit.cpp | 25 +
.../src/emitfromevents.cpp | 119 ++
.../yaml-cpp-yaml-cpp-20171024/src/emitter.cpp | 911 +++++++++++++++
.../src/emitterstate.cpp | 365 ++++++
.../src/emitterstate.h | 203 ++++
.../src/emitterutils.cpp | 483 ++++++++
.../src/emitterutils.h | 50 +
.../src/exceptions.cpp | 31 +
.../yaml-cpp-yaml-cpp-20171024/src/exp.cpp | 136 +++
thirdparty/yaml-cpp-yaml-cpp-20171024/src/exp.h | 222 ++++
.../src/indentation.h | 41 +
.../yaml-cpp-yaml-cpp-20171024/src/memory.cpp | 26 +
.../yaml-cpp-yaml-cpp-20171024/src/node.cpp | 12 +
.../src/node_data.cpp | 300 +++++
.../src/nodebuilder.cpp | 130 +++
.../src/nodebuilder.h | 70 ++
.../src/nodeevents.cpp | 101 ++
.../yaml-cpp-yaml-cpp-20171024/src/nodeevents.h | 64 ++
.../yaml-cpp-yaml-cpp-20171024/src/null.cpp | 10 +
.../src/ostream_wrapper.cpp | 57 +
.../yaml-cpp-yaml-cpp-20171024/src/parse.cpp | 72 ++
.../yaml-cpp-yaml-cpp-20171024/src/parser.cpp | 129 +++
.../yaml-cpp-yaml-cpp-20171024/src/ptr_vector.h | 43 +
.../src/regex_yaml.cpp | 45 +
.../yaml-cpp-yaml-cpp-20171024/src/regex_yaml.h | 87 ++
.../yaml-cpp-yaml-cpp-20171024/src/regeximpl.h | 186 +++
.../yaml-cpp-yaml-cpp-20171024/src/scanner.cpp | 386 +++++++
.../yaml-cpp-yaml-cpp-20171024/src/scanner.h | 190 ++++
.../src/scanscalar.cpp | 250 +++++
.../yaml-cpp-yaml-cpp-20171024/src/scanscalar.h | 63 ++
.../yaml-cpp-yaml-cpp-20171024/src/scantag.cpp | 81 ++
.../yaml-cpp-yaml-cpp-20171024/src/scantag.h | 19 +
.../src/scantoken.cpp | 437 ++++++++
.../yaml-cpp-yaml-cpp-20171024/src/setting.h | 95 ++
.../src/simplekey.cpp | 128 +++
.../src/singledocparser.cpp | 414 +++++++
.../src/singledocparser.h | 65 ++
.../yaml-cpp-yaml-cpp-20171024/src/stream.cpp | 448 ++++++++
.../yaml-cpp-yaml-cpp-20171024/src/stream.h | 76 ++
.../src/streamcharsource.h | 48 +
.../src/stringsource.h | 48 +
.../yaml-cpp-yaml-cpp-20171024/src/tag.cpp | 49 +
thirdparty/yaml-cpp-yaml-cpp-20171024/src/tag.h | 33 +
.../yaml-cpp-yaml-cpp-20171024/src/token.h | 69 ++
.../util/CMakeLists.txt | 14 +
.../yaml-cpp-yaml-cpp-20171024/util/api.cpp | 137 +++
.../yaml-cpp-yaml-cpp-20171024/util/parse.cpp | 61 +
.../yaml-cpp-yaml-cpp-20171024/util/read.cpp | 103 ++
.../yaml-cpp-yaml-cpp-20171024/util/sandbox.cpp | 36 +
.../yaml-cpp-config-version.cmake.in | 11 +
.../yaml-cpp-config.cmake.in | 14 +
.../yaml-cpp.pc.cmake | 9 +
229 files changed, 14283 insertions(+), 13749 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48416a8..bb40d27 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,11 +29,6 @@ include(FeatureSummary)
#### Establish Project Configuration ####
# Enable usage of the VERSION specifier
-# https://cmake.org/cmake/help/v3.0/policy/CMP0048.html#policy:CMP0048
-IF(POLICY CMP0048)
- CMAKE_POLICY(SET CMP0048 OLD)
-ENDIF(POLICY CMP0048)
-
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-std=c++11 " COMPILER_SUPPORTS_CXX11)
CHECK_CXX_COMPILER_FLAG("-std=c++0x " COMPILER_SUPPORTS_CXX0X)
@@ -99,10 +94,10 @@ set(JSONCPP_LIB "${JSONCPP_LIB_DIR}/lib/${prefix}jsoncpp${suffix}")
set(CIVETWEB_ENABLE_SSL_DYNAMIC_LOADING OFF CACHE BOOL "Disable dynamic SSL library loading")
set(CIVETWEB_ENABLE_CXX ON CACHE BOOL "Enable civet C++ library")
-add_subdirectory(thirdparty/yaml-cpp-yaml-cpp-0.5.3)
+add_subdirectory(thirdparty/yaml-cpp-yaml-cpp-20171024)
add_subdirectory(thirdparty/civetweb-1.9.1 EXCLUDE_FROM_ALL)
-add_subdirectory(thirdparty/libarchive-3.3.2)
include_directories(thirdparty/concurrentqueue)
+include_directories(thirdparty/yaml-cpp-yaml-cpp-20171024/include)
add_subdirectory(libminifi)
if (NOT DISABLE_CURL)
find_package(CURL)
@@ -123,12 +118,21 @@ if (NOT DISABLE_ROCKSDB)
add_subdirectory(extensions/rocksdb-repos)
mark_as_advanced(ROCKSDB-REPOS)
-
+endif()
+
+if (NOT DISABLE_LIBARCHIVE)
+ find_package(LibArchive)
+ if (NOT LibArchive_FOUND OR BUILD_LIBARCHIVE)
+ add_subdirectory(thirdparty/libarchive-3.3.2)
+ endif()
+ add_subdirectory(extensions/libarchive)
+ mark_as_advanced(ARCHIVE-EXTENSIONS)
endif()
## ENABLE FEATURE INFORMATION
ADD_FEATURE_INFO("HTTP CURL" HTTP-CURL "This enables RESTProtocol, InvokeHTTP, and the HTTPClient for Site to Site")
ADD_FEATURE_INFO("ROCKSDB REPOS" ROCKSDB-REPOS "This Enables persistent provenance, flowfile, and content repositories using RocksDB")
+ADD_FEATURE_INFO("ARCHIVE EXTENSIONS" ARCHIVE-EXTENSIONS "This Enables libarchive functionality including MergeContent")
## NOW WE CAN ADD LIBRARIES AND EXTENSIONS TO MAIN
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/cmake/BuildTests.cmake
----------------------------------------------------------------------
diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake
index 59b825d..720b00a 100644
--- a/cmake/BuildTests.cmake
+++ b/cmake/BuildTests.cmake
@@ -29,6 +29,8 @@ MACRO(GETSOURCEFILES result curdir)
SET(${result} ${dirlist})
ENDMACRO()
+find_package(Boost COMPONENTS system filesystem REQUIRED)
+
function(createTests testName)
message ("-- Adding test: ${testName}")
target_include_directories(${testName} PRIVATE BEFORE "thirdparty/catch")
@@ -51,7 +53,9 @@ function(createTests testName)
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/utils")
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/processors")
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/provenance")
- target_link_libraries(${testName} ${CMAKE_THREAD_LIBS_INIT} ${OPENSSL_LIBRARIES} minifi yaml-cpp c-library civetweb-cpp ${JSON_CPP_LIB} archive)
+ target_link_libraries(${testName} ${CMAKE_THREAD_LIBS_INIT} ${OPENSSL_LIBRARIES} core-minifi minifi yaml-cpp c-library civetweb-cpp ${JSON_CPP_LIB})
+ target_link_libraries(${testName} ${Boost_SYSTEM_LIBRARY})
+ target_link_libraries(${testName} ${Boost_FILESYSTEM_LIBRARY})
if (CURL_FOUND)
target_include_directories(${testName} PRIVATE BEFORE ${CURL_INCLUDE_DIRS})
target_include_directories(${testName} PRIVATE BEFORE "extensions/http-curl/")
@@ -77,6 +81,7 @@ GETSOURCEFILES(UNIT_TESTS "${TEST_DIR}/unit/")
GETSOURCEFILES(INTEGRATION_TESTS "${TEST_DIR}/integration/")
GETSOURCEFILES(CURL_INTEGRATION_TESTS "${TEST_DIR}/curl-tests/")
GETSOURCEFILES(ROCKSDB_INTEGRATION_TESTS "${TEST_DIR}/rocksdb-tests/")
+GETSOURCEFILES(ARCHIVE_INTEGRATION_TESTS "${TEST_DIR}/archive-tests/")
SET(UNIT_TEST_COUNT 0)
FOREACH(testfile ${UNIT_TESTS})
@@ -115,6 +120,24 @@ ENDFOREACH()
message("-- Finished building ${ROCKSDB_TEST_COUNT} RocksDB related test file(s)...")
endif(ROCKSDB-REPOS)
+if (ARCHIVE-EXTENSIONS)
+SET(EXTENSIONS_TEST_COUNT 0)
+FOREACH(testfile ${ARCHIVE_INTEGRATION_TESTS})
+ get_filename_component(testfilename "${testfile}" NAME_WE)
+ add_executable("${testfilename}" "${TEST_DIR}/archive-tests/${testfile}" ${SPD_SOURCES} "${TEST_DIR}/TestBase.cpp")
+ target_include_directories(${testfilename} PRIVATE BEFORE "extensions/libarchive")
+ target_include_directories(${testfilename} PRIVATE BEFORE "thirdparty/libarchive-3.3.2/libarchive")
+ if (APPLE)
+ target_link_libraries (${testfilename} -Wl,-all_load ${ARCHIVE-EXTENSIONS})
+ else ()
+ target_link_libraries (${testfilename} -Wl,--whole-archive ${ARCHIVE-EXTENSIONS} -Wl,--no-whole-archive)
+ endif ()
+ createTests("${testfilename}")
+ MATH(EXPR EXTENSIONS_TEST_COUNT "${ROCKSDB_TEST_COUNT}+1")
+ENDFOREACH()
+message("-- Finished building ${ARCHIVE-EXTENSIONS_TEST_COUNT} Lib Archive related test file(s)...")
+endif(ARCHIVE-EXTENSIONS)
+
if (HTTP-CURL)
SET(CURL_INT_TEST_COUNT 0)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/http-curl/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/extensions/http-curl/CMakeLists.txt b/extensions/http-curl/CMakeLists.txt
index ec56854..6989a7d 100644
--- a/extensions/http-curl/CMakeLists.txt
+++ b/extensions/http-curl/CMakeLists.txt
@@ -24,10 +24,8 @@ find_package(CURL REQUIRED)
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols")
set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols")
-include_directories(../../libminifi/include ../../libminifi/include/c2 ../../libminifi/include/c2/protocols/ ../../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/ ../../thirdparty/libarchive-3.3.2/libarchive)
-
-find_package(Boost REQUIRED)
-include_directories(${Boost_INCLUDE_DIRS})
+include_directories(../../libminifi/include ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/)
+include_directories(protocols client processors)
file(GLOB SOURCES "*.cpp" "protocols/*.cpp" "client/*.cpp" "processors/*.cpp")
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/ArchiveLoader.cpp
----------------------------------------------------------------------
diff --git a/extensions/libarchive/ArchiveLoader.cpp b/extensions/libarchive/ArchiveLoader.cpp
new file mode 100644
index 0000000..2e6949c
--- /dev/null
+++ b/extensions/libarchive/ArchiveLoader.cpp
@@ -0,0 +1,29 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ArchiveLoader.h"
+#include "core/FlowConfiguration.h"
+
+bool ArchiveFactory::added = core::FlowConfiguration::add_static_func("createArchiveFactory");
+
+extern "C" {
+
+void *createArchiveFactory(void) {
+ return new ArchiveFactory();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/ArchiveLoader.h
----------------------------------------------------------------------
diff --git a/extensions/libarchive/ArchiveLoader.h b/extensions/libarchive/ArchiveLoader.h
new file mode 100644
index 0000000..c7b1e36
--- /dev/null
+++ b/extensions/libarchive/ArchiveLoader.h
@@ -0,0 +1,67 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef EXTENSION_ARCHIVELOADER_H
+#define EXTENSION_ARCHIVELOADER_H
+
+#include "MergeContent.h"
+#include "core/ClassLoader.h"
+
+class __attribute__((visibility("default"))) ArchiveFactory : public core::ObjectFactory {
+ public:
+ ArchiveFactory() {
+
+ }
+
+ /**
+ * Gets the name of the object.
+ * @return class name of processor
+ */
+ virtual std::string getName() {
+ return "ArchiveFactory";
+ }
+
+ virtual std::string getClassName() {
+ return "ArchiveFactory";
+ }
+ /**
+ * Gets the class name for the object
+ * @return class name for the processor.
+ */
+ virtual std::vector<std::string> getClassNames() {
+ std::vector<std::string> class_names;
+ class_names.push_back("MergeContent");
+ return class_names;
+ }
+
+ virtual std::unique_ptr<ObjectFactory> assign(const std::string &class_name) {
+ std::string name = class_name;
+ std::transform(name.begin(), name.end(), name.begin(), ::tolower);
+ if (name == "mergecontent") {
+ return std::unique_ptr<ObjectFactory>(new core::DefautObjectFactory<minifi::processors::MergeContent>());
+ }
+ return nullptr;
+ }
+
+ static bool added;
+
+};
+
+extern "C" {
+void *createArchiveFactory(void);
+}
+#endif /* EXTENSION_ARCHIVELOADER_H */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/BinFiles.cpp
----------------------------------------------------------------------
diff --git a/extensions/libarchive/BinFiles.cpp b/extensions/libarchive/BinFiles.cpp
new file mode 100644
index 0000000..41df9fc
--- /dev/null
+++ b/extensions/libarchive/BinFiles.cpp
@@ -0,0 +1,304 @@
+/**
+ * @file BinFiles.cpp
+ * BinFiles class implementation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "BinFiles.h"
+#include <stdio.h>
+#include <memory>
+#include <string>
+#include <vector>
+#include <set>
+#include <queue>
+#include <map>
+#include <deque>
+#include <utility>
+#include "utils/TimeUtil.h"
+#include "utils/StringUtils.h"
+#include "core/ProcessContext.h"
+#include "core/ProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+core::Property BinFiles::MinSize("Minimum Group Size", "The minimum size of for the bundle", "0");
+core::Property BinFiles::MaxSize("Maximum Group Size", "The maximum size for the bundle. If not specified, there is no maximum.", "");
+core::Property BinFiles::MinEntries("Minimum Number of Entries", "The minimum number of files to include in a bundle", "1");
+core::Property BinFiles::MaxEntries("Maximum Number of Entries", "The maximum number of files to include in a bundle. If not specified, there is no maximum.", "");
+core::Property BinFiles::MaxBinAge("Max Bin Age", "The maximum age of a Bin that will trigger a Bin to be complete. Expected format is <duration> <time unit>", "");
+core::Property BinFiles::MaxBinCount("Maximum number of Bins", "Specifies the maximum number of bins that can be held in memory at any one time", "100");
+core::Relationship BinFiles::Original("original", "The FlowFiles that were used to create the bundle");
+core::Relationship BinFiles::Failure("failure", "If the bundle cannot be created, all FlowFiles that would have been used to created the bundle will be transferred to failure");
+const char *BinFiles::FRAGMENT_COUNT_ATTRIBUTE = "fragment.count";
+const char *BinFiles::FRAGMENT_ID_ATTRIBUTE = "fragment.identifier";
+const char *BinFiles::FRAGMENT_INDEX_ATTRIBUTE = "fragment.index";
+const char *BinFiles::SEGMENT_COUNT_ATTRIBUTE = "segment.count";
+const char *BinFiles::SEGMENT_ID_ATTRIBUTE = "segment.identifier";
+const char *BinFiles::SEGMENT_INDEX_ATTRIBUTE = "segment.index";
+const char *BinFiles::SEGMENT_ORIGINAL_FILENAME = "segment.original.filename";
+const char *BinFiles::TAR_PERMISSIONS_ATTRIBUTE = "tar.permissions";
+
+void BinFiles::initialize() {
+ // Set the supported properties
+ std::set<core::Property> properties;
+ properties.insert(MinSize);
+ properties.insert(MaxSize);
+ properties.insert(MinEntries);
+ properties.insert(MaxEntries);
+ properties.insert(MaxBinAge);
+ properties.insert(MaxBinCount);
+ setSupportedProperties(properties);
+ // Set the supported relationships
+ std::set<core::Relationship> relationships;
+ relationships.insert(Original);
+ relationships.insert(Failure);
+ setSupportedRelationships(relationships);
+}
+
+void BinFiles::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) {
+ std::string value;
+ int64_t valInt;
+ if (context->getProperty(MinSize.getName(), value) && !value.empty() && core::Property::StringToInt(value, valInt)) {
+ this->binManager_.setMinSize(valInt);
+ logger_->log_info("BinFiles: MinSize [%d]", valInt);
+ }
+ value = "";
+ if (context->getProperty(MaxSize.getName(), value) && !value.empty() && core::Property::StringToInt(value, valInt)) {
+ this->binManager_.setMaxSize(valInt);
+ logger_->log_info("BinFiles: MaxSize [%d]", valInt);
+ }
+ value = "";
+ if (context->getProperty(MinEntries.getName(), value) && !value.empty() && core::Property::StringToInt(value, valInt)) {
+ this->binManager_.setMinEntries(valInt);
+ logger_->log_info("BinFiles: MinEntries [%d]", valInt);
+ }
+ value = "";
+ if (context->getProperty(MaxEntries.getName(), value) && !value.empty() && core::Property::StringToInt(value, valInt)) {
+ this->binManager_.setMaxEntries(valInt);
+ logger_->log_info("BinFiles: MaxEntries [%d]", valInt);
+ }
+ value = "";
+ if (context->getProperty(MaxBinCount.getName(), value) && !value.empty() && core::Property::StringToInt(value, valInt)) {
+ maxBinCount_ = static_cast<int> (valInt);
+ logger_->log_info("BinFiles: MaxBinCount [%d]", valInt);
+ }
+ value = "";
+ if (context->getProperty(MaxBinAge.getName(), value) && !value.empty()) {
+ core::TimeUnit unit;
+ if (core::Property::StringToTime(value, valInt, unit) && core::Property::ConvertTimeUnitToMS(valInt, unit, valInt)) {
+ this->binManager_.setBinAge(valInt);
+ logger_->log_info("BinFiles: MaxBinAge [%d]", valInt);
+ }
+ }
+}
+
+void BinFiles::preprocessFlowFile(core::ProcessContext *context, core::ProcessSession *session, std::shared_ptr<core::FlowFile> flow) {
+ // handle backward compatibility with old segment attributes
+ std::string value;
+ if (!flow->getAttribute(BinFiles::FRAGMENT_COUNT_ATTRIBUTE, value) && flow->getAttribute(BinFiles::SEGMENT_COUNT_ATTRIBUTE, value)) {
+ flow->setAttribute(BinFiles::FRAGMENT_COUNT_ATTRIBUTE, value);
+ }
+ if (!flow->getAttribute(BinFiles::FRAGMENT_INDEX_ATTRIBUTE, value) && flow->getAttribute(BinFiles::SEGMENT_INDEX_ATTRIBUTE, value)) {
+ flow->setAttribute(BinFiles::FRAGMENT_INDEX_ATTRIBUTE, value);
+ }
+ if (!flow->getAttribute(BinFiles::FRAGMENT_ID_ATTRIBUTE, value) && flow->getAttribute(BinFiles::SEGMENT_ID_ATTRIBUTE, value)) {
+ flow->setAttribute(BinFiles::FRAGMENT_ID_ATTRIBUTE, value);
+ }
+}
+
+void BinManager::gatherReadyBins() {
+ std::lock_guard < std::mutex > lock(mutex_);
+ std::vector< std::string > emptyQueue;
+ for (std::map<std::string, std::unique_ptr<std::deque<std::unique_ptr<Bin>>> >::iterator it=groupBinMap_.begin(); it !=groupBinMap_.end(); ++it) {
+ std::unique_ptr < std::deque<std::unique_ptr<Bin>>>&queue = it->second;
+ while (!queue->empty()) {
+ std::unique_ptr<Bin> &bin = queue->front();
+ if (bin->isReadyForMerge() || (binAge_ != ULLONG_MAX && bin->isOlderThan(binAge_))) {
+ readyBin_.push_back(std::move(bin));
+ queue->pop_front();
+ binCount_--;
+ logger_->log_info("BinManager move bin %s to ready bins for group %s", readyBin_.back()->getUUIDStr(), readyBin_.back()->getGroupId());
+ } else {
+ break;
+ }
+ }
+ if (queue->empty()) {
+ emptyQueue.push_back(it->first);
+ }
+ }
+ for (auto group : emptyQueue) {
+ // erase from the map if the queue is empty for the group
+ groupBinMap_.erase(group);
+ }
+ logger_->log_info("BinManager groupBinMap size %d", groupBinMap_.size());
+}
+
+void BinManager::removeOldestBin() {
+ std::lock_guard < std::mutex > lock(mutex_);
+ uint64_t olddate = ULLONG_MAX;
+ std::unique_ptr < std::deque<std::unique_ptr<Bin>>>*oldqueue;
+ for (std::map<std::string, std::unique_ptr<std::deque<std::unique_ptr<Bin>>>>::iterator it=groupBinMap_.begin(); it !=groupBinMap_.end(); ++it) {
+ std::unique_ptr < std::deque<std::unique_ptr<Bin>>>&queue = it->second;
+ if (!queue->empty()) {
+ std::unique_ptr<Bin> &bin = queue->front();
+ if (bin->getBinAge() < olddate) {
+ olddate = bin->getBinAge();
+ oldqueue = &queue;
+ }
+ }
+ }
+ if (olddate != ULLONG_MAX) {
+ std::unique_ptr<Bin> &remove = (*oldqueue)->front();
+ std::string group = remove->getGroupId();
+ readyBin_.push_back(std::move(remove));
+ (*oldqueue)->pop_front();
+ binCount_--;
+ logger_->log_info("BinManager move bin %s to ready bins for group %s", readyBin_.back()->getUUIDStr(), readyBin_.back()->getGroupId());
+ if ((*oldqueue)->empty()) {
+ groupBinMap_.erase(group);
+ }
+ }
+ logger_->log_info("BinManager groupBinMap size %d", groupBinMap_.size());
+}
+
+void BinManager::getReadyBin(std::deque<std::unique_ptr<Bin>> &retBins) {
+ std::lock_guard < std::mutex > lock(mutex_);
+ while (!readyBin_.empty()) {
+ std::unique_ptr<Bin> &bin = readyBin_.front();
+ retBins.push_back(std::move(bin));
+ readyBin_.pop_front();
+ }
+}
+
+bool BinManager::offer(const std::string &group, std::shared_ptr<core::FlowFile> flow) {
+ std::lock_guard < std::mutex > lock(mutex_);
+ if (flow->getSize() > maxSize_) {
+ // could not be added to a bin -- too large by itself, so create a separate bin for just this guy.
+ std::unique_ptr<Bin> bin = std::unique_ptr < Bin > (new Bin(0, ULLONG_MAX, 1, INT_MAX, "", group));
+ if (!bin->offer(flow))
+ return false;
+ readyBin_.push_back(std::move(bin));
+ logger_->log_info("BinManager move bin %s to ready bins for group %s", readyBin_.back()->getUUIDStr(), group);
+ return true;
+ }
+ auto search = groupBinMap_.find(group);
+ if (search != groupBinMap_.end()) {
+ std::unique_ptr < std::deque<std::unique_ptr<Bin>>>&queue = search->second;
+ if (!queue->empty()) {
+ std::unique_ptr<Bin> &tail = queue->back();
+ if (!tail->offer(flow)) {
+ // last bin can not offer the flow
+ std::unique_ptr<Bin> bin = std::unique_ptr < Bin > (new Bin(minSize_, maxSize_, minEntries_, maxEntries_, fileCount_, group));
+ if (!bin->offer(flow))
+ return false;
+ queue->push_back(std::move(bin));
+ logger_->log_info("BinManager add bin %s to group %s", queue->back()->getUUIDStr(), group);
+ binCount_++;
+ }
+ } else {
+ std::unique_ptr<Bin> bin = std::unique_ptr < Bin > (new Bin(minSize_, maxSize_, minEntries_, maxEntries_, fileCount_, group));
+ if (!bin->offer(flow))
+ return false;
+ queue->push_back(std::move(bin));
+ binCount_++;
+ logger_->log_info("BinManager add bin %s to group %s", queue->back()->getUUIDStr(), group);
+ }
+ } else {
+ std::unique_ptr<std::deque<std::unique_ptr<Bin>>> queue = std::unique_ptr<std::deque<std::unique_ptr<Bin>>> (new std::deque<std::unique_ptr<Bin>>());
+ std::unique_ptr<Bin> bin = std::unique_ptr < Bin > (new Bin(minSize_, maxSize_, minEntries_, maxEntries_, fileCount_, group));
+ if (!bin->offer(flow))
+ return false;
+ queue->push_back(std::move(bin));
+ logger_->log_info("BinManager add bin %s to group %s", queue->back()->getUUIDStr(), group);
+ groupBinMap_.insert(std::make_pair(group, std::move(queue)));
+ binCount_++;
+ }
+
+ return true;
+}
+
+void BinFiles::onTrigger(std::shared_ptr<core::ProcessContext> context, std::shared_ptr<core::ProcessSession> session) {
+ std::shared_ptr<FlowFileRecord> flow = std::static_pointer_cast < FlowFileRecord > (session->get());
+
+ if (flow != nullptr) {
+ preprocessFlowFile(context.get(), session.get(), flow);
+ std::string groupId = getGroupId(context.get(), flow);
+
+ bool offer = this->binManager_.offer(groupId, flow);
+ if (!offer) {
+ session->transfer(flow, Failure);
+ context->yield();
+ return;
+ }
+
+ // remove the flowfile from the process session, it add to merge session later.
+ session->remove(flow);
+ }
+
+ // migrate bin to ready bin
+ this->binManager_.gatherReadyBins();
+ if (this->binManager_.getBinCount() > maxBinCount_) {
+ // bin count reach max allowed
+ context->yield();
+ logger_->log_info("BinFiles reach max bin count %d", this->binManager_.getBinCount());
+ this->binManager_.removeOldestBin();
+ }
+
+ // get the ready bin
+ std::deque<std::unique_ptr<Bin>> readyBins;
+ binManager_.getReadyBin(readyBins);
+
+ // process the ready bin
+ if (!readyBins.empty()) {
+ // create session for merge
+ core::ProcessSession mergeSession(context);
+ while (!readyBins.empty()) {
+ std::unique_ptr<Bin> bin = std::move(readyBins.front());
+ readyBins.pop_front();
+ // add bin's flows to the session
+ this->addFlowsToSession(context.get(), &mergeSession, bin);
+ logger_->log_info("BinFiles start to process bin %s for group %s", bin->getUUIDStr(), bin->getGroupId());
+ if (!this->processBin(context.get(), &mergeSession, bin))
+ this->transferFlowsToFail(context.get(), &mergeSession, bin);
+ }
+ mergeSession.commit();
+ }
+}
+
+void BinFiles::transferFlowsToFail(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin) {
+ std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
+ for (auto flow : flows) {
+ session->transfer(flow, Failure);
+ }
+ flows.clear();
+}
+
+void BinFiles::addFlowsToSession(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin) {
+ std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
+ for (auto flow : flows) {
+ session->add(flow);
+ }
+}
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/BinFiles.h
----------------------------------------------------------------------
diff --git a/extensions/libarchive/BinFiles.h b/extensions/libarchive/BinFiles.h
new file mode 100644
index 0000000..240fb09
--- /dev/null
+++ b/extensions/libarchive/BinFiles.h
@@ -0,0 +1,308 @@
+/**
+ * @file BinFiles.h
+ * BinFiles class declaration
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __BIN_FILES_H__
+#define __BIN_FILES_H__
+
+#include <climits>
+#include <deque>
+#include <map>
+#include "FlowFileRecord.h"
+#include "core/Processor.h"
+#include "core/ProcessSession.h"
+#include "core/Core.h"
+#include "core/Resource.h"
+#include "core/logging/LoggerConfiguration.h"
+#include "utils/Id.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+// Bin Class
+class Bin {
+ public:
+ // Constructor
+ /*!
+ * Create a new Bin. Note: this object is not thread safe
+ */
+ explicit Bin(const uint64_t &minSize, const uint64_t &maxSize, const int &minEntries, const int & maxEntries, const std::string &fileCount, const std::string &groupId)
+ : minSize_(minSize),
+ maxSize_(maxSize),
+ maxEntries_(maxEntries),
+ minEntries_(minEntries),
+ fileCount_(fileCount),
+ groupId_(groupId),
+ logger_(logging::LoggerFactory<Bin>::getLogger()) {
+ queued_data_size_ = 0;
+ creation_dated_ = getTimeMillis();
+ std::shared_ptr<utils::IdGenerator> id_generator = utils::IdGenerator::getIdGenerator();
+ char uuidStr[37] = { 0 };
+ id_generator->generate(uuid_);
+ uuid_unparse_lower(uuid_, uuidStr);
+ uuid_str_ = uuidStr;
+ logger_->log_info("Bin %s for group %s created", uuid_str_, groupId_);
+ }
+ virtual ~Bin() {
+ logger_->log_info("Bin %s for group %s destroyed", uuid_str_, groupId_);
+ }
+ // check whether the bin is full
+ bool isFull() {
+ if (queued_data_size_ >= maxSize_ || queue_.size() >= maxEntries_)
+ return true;
+ else
+ return false;
+ }
+ // check whether the bin meet the min required size and entries so that it can be processed for merge
+ bool isReadyForMerge() {
+ return isFull() || (queued_data_size_ >= minSize_ && queue_.size() >= minEntries_);
+ }
+ // check whether the bin is older than the time specified in msec
+ bool isOlderThan(const uint64_t &duration) {
+ uint64_t currentTime = getTimeMillis();
+ if (currentTime > (creation_dated_ + duration))
+ return true;
+ else
+ return false;
+ }
+ std::deque<std::shared_ptr<core::FlowFile>> & getFlowFile() {
+ return queue_;
+ }
+ // offer the flowfile to the bin
+ bool offer(std::shared_ptr<core::FlowFile> flow) {
+ if (!fileCount_.empty()) {
+ std::string value;
+ if (flow->getAttribute(fileCount_, value)) {
+ try {
+ // for defrag case using the identification
+ int count = std::stoi(value);
+ maxEntries_ = count;
+ minEntries_ = count;
+ } catch (...) {
+
+ }
+ }
+ }
+
+ if ((queued_data_size_ + flow->getSize()) > maxSize_ || (queue_.size() + 1) > maxEntries_)
+ return false;
+
+ queue_.push_back(flow);
+ queued_data_size_ += flow->getSize();
+ logger_->log_info("Bin %s for group %s offer size %d byte %d min_entry %d max_entry %d", uuid_str_, groupId_, queue_.size(), queued_data_size_, minEntries_, maxEntries_);
+
+ return true;
+ }
+ // getBinAge
+ uint64_t getBinAge() {
+ return creation_dated_;
+ }
+ int getSize() {
+ return queue_.size();
+ }
+ // Get the UUID as string
+ std::string getUUIDStr() {
+ return uuid_str_;
+ }
+ std::string getGroupId() {
+ return groupId_;
+ }
+
+ protected:
+
+ private:
+ uint64_t minSize_;
+ uint64_t maxSize_;
+ int maxEntries_;
+ int minEntries_;
+ // Queued data size
+ uint64_t queued_data_size_;
+ // Queue for the Flow File
+ std::deque<std::shared_ptr<core::FlowFile>> queue_;
+ uint64_t creation_dated_;
+ std::string fileCount_;
+ std::string groupId_;
+ std::shared_ptr<logging::Logger> logger_;
+ // A global unique identifier
+ uuid_t uuid_;
+ // UUID string
+ std::string uuid_str_;
+};
+
+// BinManager Class
+class BinManager {
+ public:
+ // Constructor
+ /*!
+ * Create a new BinManager
+ */
+ BinManager()
+ : minSize_(0),
+ maxSize_(ULLONG_MAX),
+ maxEntries_(INT_MAX),
+ minEntries_(1),
+ binAge_(ULLONG_MAX),
+ binCount_(0),
+ logger_(logging::LoggerFactory<BinManager>::getLogger()) {
+ }
+ virtual ~BinManager() {
+ purge();
+ }
+ void setMinSize(const uint64_t &size) {
+ minSize_ = size;
+ }
+ void setMaxSize(const uint64_t &size) {
+ maxSize_ = size;
+ }
+ void setMaxEntries(const int &entries) {
+ maxEntries_ = entries;
+ }
+ void setMinEntries(const int &entries) {
+ minEntries_ = entries;
+ }
+ void setBinAge(const uint64_t &age) {
+ binAge_ = age;
+ }
+ int getBinCount() {
+ return binCount_;
+ }
+ void setFileCount(const std::string &value) {
+ fileCount_ = value;
+ }
+ void purge() {
+ std::lock_guard<std::mutex> lock(mutex_);
+ groupBinMap_.clear();
+ binCount_ = 0;
+ }
+ // Adds the given flowFile to the first available bin in which it fits for the given group or creates a new bin in the specified group if necessary.
+ bool offer(const std::string &group, std::shared_ptr<core::FlowFile> flow);
+ // gather ready bins once the bin are full enough or exceed bin age
+ void gatherReadyBins();
+ // remove oldest bin
+ void removeOldestBin();
+ // get ready bin from binManager
+ void getReadyBin(std::deque<std::unique_ptr<Bin>> &retBins);
+
+ protected:
+
+ private:
+ std::mutex mutex_;
+ uint64_t minSize_;
+ uint64_t maxSize_;
+ int maxEntries_;
+ int minEntries_;
+ std::string fileCount_;
+ // Bin Age in msec
+ uint64_t binAge_;
+ std::map<std::string, std::unique_ptr<std::deque<std::unique_ptr<Bin>>> >groupBinMap_;
+ std::deque<std::unique_ptr<Bin>> readyBin_;
+ int binCount_;
+ std::shared_ptr<logging::Logger> logger_;
+};
+
+// BinFiles Class
+class BinFiles : public core::Processor {
+ public:
+ // Constructor
+ /*!
+ * Create a new processor
+ */
+ explicit BinFiles(std::string name, uuid_t uuid = NULL)
+ : core::Processor(name, uuid),
+ logger_(logging::LoggerFactory<BinFiles>::getLogger()) {
+ maxBinCount_ = 100;
+ }
+ // Destructor
+ virtual ~BinFiles() {
+ }
+ // Processor Name
+ static constexpr char const* ProcessorName = "BinFiles";
+ // Supported Properties
+ static core::Property MinSize;
+ static core::Property MaxSize;
+ static core::Property MinEntries;
+ static core::Property MaxEntries;
+ static core::Property MaxBinCount;
+ static core::Property MaxBinAge;
+
+ // Supported Relationships
+ static core::Relationship Failure;
+ static core::Relationship Original;
+
+ // attributes
+ static const char *FRAGMENT_ID_ATTRIBUTE;
+ static const char *FRAGMENT_INDEX_ATTRIBUTE;
+ static const char *FRAGMENT_COUNT_ATTRIBUTE;
+
+ static const char *SEGMENT_ID_ATTRIBUTE;
+ static const char *SEGMENT_INDEX_ATTRIBUTE;
+ static const char *SEGMENT_COUNT_ATTRIBUTE;
+ static const char *SEGMENT_ORIGINAL_FILENAME;
+ static const char *TAR_PERMISSIONS_ATTRIBUTE;
+
+ public:
+ /**
+ * Function that's executed when the processor is scheduled.
+ * @param context process context.
+ * @param sessionFactory process session factory that is used when creating
+ * ProcessSession objects.
+ */
+ void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory);
+ // OnTrigger method, implemented by NiFi BinFiles
+ virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session) {
+ }
+ // OnTrigger method, implemented by NiFi BinFiles
+ virtual void onTrigger(std::shared_ptr<core::ProcessContext> context, std::shared_ptr<core::ProcessSession> session);
+ // Initialize, over write by NiFi BinFiles
+ virtual void initialize(void);
+
+ protected:
+ // Allows general pre-processing of a flow file before it is offered to a bin. This is called before getGroupId().
+ virtual void preprocessFlowFile(core::ProcessContext *context, core::ProcessSession *session, std::shared_ptr<core::FlowFile> flow);
+ // Returns a group ID representing a bin. This allows flow files to be binned into like groups
+ virtual std::string getGroupId(core::ProcessContext *context, std::shared_ptr<core::FlowFile> flow) {
+ return "";
+ }
+ // Processes a single bin.
+ virtual bool processBin(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin) {
+ return false;
+ }
+ // transfer flows to failure in bin
+ void transferFlowsToFail(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin);
+ // add flows to session
+ void addFlowsToSession(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin);
+
+ BinManager binManager_;
+
+ private:
+ std::shared_ptr<logging::Logger> logger_;
+ int maxBinCount_;
+};
+
+REGISTER_RESOURCE(BinFiles);
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/extensions/libarchive/CMakeLists.txt b/extensions/libarchive/CMakeLists.txt
new file mode 100644
index 0000000..5d982f4
--- /dev/null
+++ b/extensions/libarchive/CMakeLists.txt
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cmake_minimum_required(VERSION 2.6)
+
+
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols")
+set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols")
+
+include_directories(../../libminifi/include ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/)
+
+include_directories(../../thirdparty/libarchive-3.3.2/libarchive)
+include_directories(${Boost_INCLUDE_DIRS})
+
+file(GLOB SOURCES "*.cpp")
+
+add_library(minifi-archive-extensions STATIC ${SOURCES})
+set_property(TARGET minifi-archive-extensions PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(THREADS_HAVE_PTHREAD_ARG)
+ target_compile_options(PUBLIC minifi-archive-extensions "-pthread")
+endif()
+if(CMAKE_THREAD_LIBS_INIT)
+ target_link_libraries(minifi-archive-extensions "${CMAKE_THREAD_LIBS_INIT}")
+endif()
+
+
+# Include UUID
+find_package(UUID REQUIRED)
+target_link_libraries(minifi-archive-extensions ${LIBMINIFI} ${UUID_LIBRARIES} ${JSONCPP_LIB})
+add_dependencies(minifi-archive-extensions jsoncpp_project)
+find_package(OpenSSL REQUIRED)
+include_directories(${OPENSSL_INCLUDE_DIR})
+target_link_libraries(minifi-archive-extensions ${CMAKE_DL_LIBS} )
+if (LibArchive_FOUND AND NOT BUILD_LIBARCHIVE)
+ target_link_libraries(minifi-archive-extensions ${LibArchive_LIBRARIES} )
+else()
+ target_link_libraries(minifi-archive-extensions archive_static )
+endif()
+find_package(ZLIB REQUIRED)
+include_directories(${ZLIB_INCLUDE_DIRS})
+target_link_libraries (minifi-archive-extensions ${ZLIB_LIBRARIES})
+if (WIN32)
+ set_target_properties(minifi-archive-extensions PROPERTIES
+ LINK_FLAGS "/WHOLEARCHIVE"
+ )
+elseif (APPLE)
+ set_target_properties(minifi-archive-extensions PROPERTIES
+ LINK_FLAGS "-Wl,-all_load"
+ )
+else ()
+ set_target_properties(minifi-archive-extensions PROPERTIES
+ LINK_FLAGS "-Wl,--whole-archive"
+ )
+endif ()
+
+
+SET (ARCHIVE-EXTENSIONS minifi-archive-extensions PARENT_SCOPE)
+
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/MergeContent.cpp
----------------------------------------------------------------------
diff --git a/extensions/libarchive/MergeContent.cpp b/extensions/libarchive/MergeContent.cpp
new file mode 100644
index 0000000..91f542b
--- /dev/null
+++ b/extensions/libarchive/MergeContent.cpp
@@ -0,0 +1,334 @@
+/**
+ * @file MergeContent.cpp
+ * MergeContent class implementation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "MergeContent.h"
+#include <stdio.h>
+#include <memory>
+#include <string>
+#include <vector>
+#include <set>
+#include <queue>
+#include <map>
+#include <deque>
+#include <utility>
+#include <algorithm>
+#include "utils/TimeUtil.h"
+#include "utils/StringUtils.h"
+#include "core/ProcessContext.h"
+#include "core/ProcessSession.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+core::Property MergeContent::MergeStrategy("Merge Strategy", "Defragment or Bin-Packing Algorithm", MERGE_STRATEGY_DEFRAGMENT);
+core::Property MergeContent::MergeFormat("Merge Format", "Merge Format", MERGE_FORMAT_CONCAT_VALUE);
+core::Property MergeContent::CorrelationAttributeName("Correlation Attribute Name", "Correlation Attribute Name", "");
+core::Property MergeContent::DelimiterStratgey("Delimiter Strategy", "Determines if Header, Footer, and Demarcator should point to files", DELIMITER_STRATEGY_FILENAME);
+core::Property MergeContent::Header("Header File", "Filename specifying the header to use", "");
+core::Property MergeContent::Footer("Footer File", "Filename specifying the footer to use", "");
+core::Property MergeContent::Demarcator("Demarcator File", "Filename specifying the demarcator to use", "");
+core::Property MergeContent::KeepPath("Keep Path", "If using the Zip or Tar Merge Format, specifies whether or not the FlowFiles' paths should be included in their entry", "false");
+core::Relationship MergeContent::Merge("merged", "The FlowFile containing the merged content");
+const char *BinaryConcatenationMerge::mimeType = "application/octet-stream";
+const char *TarMerge::mimeType = "application/tar";
+const char *ZipMerge::mimeType = "application/zip";
+
+void MergeContent::initialize() {
+ // Set the supported properties
+ std::set<core::Property> properties;
+ properties.insert(MinSize);
+ properties.insert(MaxSize);
+ properties.insert(MinEntries);
+ properties.insert(MaxEntries);
+ properties.insert(MaxBinAge);
+ properties.insert(MaxBinCount);
+ properties.insert(MergeStrategy);
+ properties.insert(MergeFormat);
+ properties.insert(CorrelationAttributeName);
+ properties.insert(DelimiterStratgey);
+ properties.insert(Header);
+ properties.insert(Footer);
+ properties.insert(Demarcator);
+ properties.insert(KeepPath);
+ setSupportedProperties(properties);
+ // Set the supported relationships
+ std::set<core::Relationship> relationships;
+ relationships.insert(Original);
+ relationships.insert(Failure);
+ relationships.insert(Merge);
+ setSupportedRelationships(relationships);
+}
+
+std::string MergeContent::readContent(std::string path) {
+ std::string contents;
+ std::ifstream in(path.c_str(), std::ios::in | std::ios::binary);
+ if (in) {
+ in.seekg(0, std::ios::end);
+ contents.resize(in.tellg());
+ in.seekg(0, std::ios::beg);
+ in.read(&contents[0], contents.size());
+ in.close();
+ }
+ return (contents);
+}
+
+void MergeContent::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) {
+ std::string value;
+ BinFiles::onSchedule(context, sessionFactory);
+ if (context->getProperty(MergeStrategy.getName(), value) && !value.empty()) {
+ this->mergeStratgey_ = value;
+ }
+ value = "";
+ if (context->getProperty(MergeFormat.getName(), value) && !value.empty()) {
+ this->mergeFormat_ = value;
+ }
+ value = "";
+ if (context->getProperty(CorrelationAttributeName.getName(), value) && !value.empty()) {
+ this->correlationAttributeName_ = value;
+ }
+ value = "";
+ if (context->getProperty(DelimiterStratgey.getName(), value) && !value.empty()) {
+ this->delimiterStratgey_ = value;
+ }
+ value = "";
+ if (context->getProperty(Header.getName(), value) && !value.empty()) {
+ this->header_ = value;
+ }
+ value = "";
+ if (context->getProperty(Footer.getName(), value) && !value.empty()) {
+ this->footer_ = value;
+ }
+ value = "";
+ if (context->getProperty(Demarcator.getName(), value) && !value.empty()) {
+ this->demarcator_ = value;
+ }
+ value = "";
+ if (context->getProperty(KeepPath.getName(), value) && !value.empty()) {
+ org::apache::nifi::minifi::utils::StringUtils::StringToBool(value, keepPath_);
+ }
+ if (mergeStratgey_ == MERGE_STRATEGY_DEFRAGMENT) {
+ binManager_.setFileCount(FRAGMENT_COUNT_ATTRIBUTE);
+ }
+ logger_->log_info("Merge Content: Strategy [%s] Format [%s] Correlation Attribute [%s] Delimiter [%s]", mergeStratgey_, mergeFormat_, correlationAttributeName_, delimiterStratgey_);
+ logger_->log_info("Merge Content: Footer [%s] Header [%s] Demarcator [%s] KeepPath [%d]", footer_, header_, demarcator_, keepPath_);
+ if (delimiterStratgey_ == DELIMITER_STRATEGY_FILENAME) {
+ if (!header_.empty()) {
+ this->headerContent_ = readContent(header_);
+ }
+ if (!footer_.empty()) {
+ this->footerContent_ = readContent(footer_);
+ }
+ if (!demarcator_.empty()) {
+ this->demarcatorContent_ = readContent(demarcator_);
+ }
+ }
+ if (delimiterStratgey_ == DELIMITER_STRATEGY_TEXT) {
+ this->headerContent_ = header_;
+ this->footerContent_ = footer_;
+ this->demarcatorContent_ = demarcator_;
+ }
+}
+
+std::string MergeContent::getGroupId(core::ProcessContext *context, std::shared_ptr<core::FlowFile> flow) {
+ std::string groupId = "";
+ std::string value;
+ if (!correlationAttributeName_.empty()) {
+ if (flow->getAttribute(correlationAttributeName_, value))
+ groupId = value;
+ }
+ if (groupId.empty() && mergeStratgey_ == MERGE_STRATEGY_DEFRAGMENT) {
+ if (flow->getAttribute(FRAGMENT_ID_ATTRIBUTE, value))
+ groupId = value;
+ }
+ return groupId;
+}
+
+bool MergeContent::checkDefragment(std::unique_ptr<Bin> &bin) {
+ std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
+ if (!flows.empty()) {
+ std::shared_ptr<core::FlowFile> front = flows.front();
+ std::string fragId;
+ if (!front->getAttribute(BinFiles::FRAGMENT_ID_ATTRIBUTE, fragId))
+ return false;
+ std::string fragCount;
+ if (!front->getAttribute(BinFiles::FRAGMENT_COUNT_ATTRIBUTE, fragCount))
+ return false;
+ int fragCountInt;
+ try {
+ fragCountInt = std::stoi(fragCount);
+ }
+ catch (...) {
+ return false;
+ }
+ for (auto flow : flows) {
+ std::string value;
+ if (!flow->getAttribute(BinFiles::FRAGMENT_ID_ATTRIBUTE, value))
+ return false;
+ if (value != fragId)
+ return false;
+ if (!flow->getAttribute(BinFiles::FRAGMENT_COUNT_ATTRIBUTE, value))
+ return false;
+ if (value != fragCount)
+ return false;
+ if (!flow->getAttribute(BinFiles::FRAGMENT_INDEX_ATTRIBUTE, value))
+ return false;
+ try {
+ int index = std::stoi(value);
+ if (index < 0 || index >= fragCountInt)
+ return false;
+ }
+ catch (...) {
+ return false;
+ }
+ }
+ } else {
+ return false;
+ }
+
+ return true;
+}
+
+void MergeContent::onTrigger(core::ProcessContext *context, core::ProcessSession *session) {
+ BinFiles::onTrigger(context, session);
+}
+
+bool MergeContent::processBin(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin) {
+ if (mergeStratgey_ != MERGE_STRATEGY_DEFRAGMENT && mergeStratgey_ != MERGE_STRATEGY_BIN_PACK)
+ return false;
+
+ if (mergeStratgey_ == MERGE_STRATEGY_DEFRAGMENT) {
+ // check the flowfile fragment values
+ if (!checkDefragment(bin)) {
+ logger_->log_error("Merge Content check defgrament failed");
+ return false;
+ }
+ // sort the flowfile fragment index
+ std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
+ std::sort(flows.begin(), flows.end(), [] (const std::shared_ptr<core::FlowFile> &first, const std::shared_ptr<core::FlowFile> &second)
+ {std::string value;
+ first->getAttribute(BinFiles::FRAGMENT_INDEX_ATTRIBUTE, value);
+ int indexFirst = std::stoi(value);
+ second->getAttribute(BinFiles::FRAGMENT_INDEX_ATTRIBUTE, value);
+ int indexSecond = std::stoi(value);
+ if (indexSecond > indexFirst)
+ return true;
+ else
+ return false;
+ });
+ }
+
+ std::unique_ptr<MergeBin> mergeBin;
+ if (mergeFormat_ == MERGE_FORMAT_CONCAT_VALUE || mergeFormat_ == MERGE_FORMAT_TAR_VALUE
+ || mergeFormat_ == MERGE_FORMAT_ZIP_VALUE) {
+ if (mergeFormat_ == MERGE_FORMAT_CONCAT_VALUE)
+ mergeBin = std::unique_ptr < MergeBin > (new BinaryConcatenationMerge());
+ else if (mergeFormat_ == MERGE_FORMAT_TAR_VALUE)
+ mergeBin = std::unique_ptr < MergeBin > (new TarMerge());
+ else if (mergeFormat_ == MERGE_FORMAT_ZIP_VALUE)
+ mergeBin = std::unique_ptr < MergeBin > (new ZipMerge());
+ else
+ return false;
+
+ std::shared_ptr<core::FlowFile> mergeFlow;
+ try {
+ mergeFlow = mergeBin->merge(context, session, bin->getFlowFile(), this->headerContent_, this->footerContent_, this->demarcatorContent_);
+ } catch (...) {
+ logger_->log_error("Merge Content merge catch exception");
+ return false;
+ }
+ session->putAttribute(mergeFlow, BinFiles::FRAGMENT_COUNT_ATTRIBUTE, std::to_string(bin->getSize()));
+ // we successfully merge the flow
+ session->transfer(mergeFlow, Merge);
+ std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
+ for (auto flow : flows) {
+ session->transfer(flow, Original);
+ }
+ logger_->log_info("Merge FlowFile record UUID %s, payload length %d", mergeFlow->getUUIDStr(), mergeFlow->getSize());
+ } else {
+ logger_->log_error("Merge format not supported %s", mergeFormat_);
+ return false;
+ }
+ return true;
+}
+
+std::shared_ptr<core::FlowFile> BinaryConcatenationMerge::merge(core::ProcessContext *context, core::ProcessSession *session,
+ std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header, std::string &footer, std::string &demarcator) {
+ std::shared_ptr<FlowFileRecord> flowFile = std::static_pointer_cast < FlowFileRecord > (session->create());
+ BinaryConcatenationMerge::WriteCallback callback(header, footer, demarcator, flows, session);
+ session->write(flowFile, &callback);
+ session->putAttribute(flowFile, FlowAttributeKey(MIME_TYPE), this->getMergedContentType());
+ std::string fileName;
+ if (flows.size() == 1) {
+ flows.front()->getAttribute(FlowAttributeKey(FILENAME), fileName);
+ } else {
+ flows.front()->getAttribute(BinFiles::SEGMENT_ORIGINAL_FILENAME, fileName);
+ }
+ if (!fileName.empty())
+ session->putAttribute(flowFile, FlowAttributeKey(FILENAME), fileName);
+ return flowFile;
+}
+
+std::shared_ptr<core::FlowFile> TarMerge::merge(core::ProcessContext *context, core::ProcessSession *session, std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header,
+ std::string &footer, std::string &demarcator) {
+ std::shared_ptr<FlowFileRecord> flowFile = std::static_pointer_cast < FlowFileRecord > (session->create());
+ ArchiveMerge::WriteCallback callback(std::string(MERGE_FORMAT_TAR_VALUE), flows, session);
+ session->write(flowFile, &callback);
+ session->putAttribute(flowFile, FlowAttributeKey(MIME_TYPE), this->getMergedContentType());
+ std::string fileName;
+ flowFile->getAttribute(FlowAttributeKey(FILENAME), fileName);
+ if (flows.size() == 1) {
+ flows.front()->getAttribute(FlowAttributeKey(FILENAME), fileName);
+ } else {
+ flows.front()->getAttribute(BinFiles::SEGMENT_ORIGINAL_FILENAME, fileName);
+ }
+ if (!fileName.empty()) {
+ fileName += ".tar";
+ session->putAttribute(flowFile, FlowAttributeKey(FILENAME), fileName);
+ }
+ return flowFile;
+}
+
+std::shared_ptr<core::FlowFile> ZipMerge::merge(core::ProcessContext *context, core::ProcessSession *session, std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header,
+ std::string &footer, std::string &demarcator) {
+ std::shared_ptr<FlowFileRecord> flowFile = std::static_pointer_cast < FlowFileRecord > (session->create());
+ ArchiveMerge::WriteCallback callback(std::string(MERGE_FORMAT_ZIP_VALUE), flows, session);
+ session->write(flowFile, &callback);
+ session->putAttribute(flowFile, FlowAttributeKey(MIME_TYPE), this->getMergedContentType());
+ std::string fileName;
+ flowFile->getAttribute(FlowAttributeKey(FILENAME), fileName);
+ if (flows.size() == 1) {
+ flows.front()->getAttribute(FlowAttributeKey(FILENAME), fileName);
+ } else {
+ flows.front()->getAttribute(BinFiles::SEGMENT_ORIGINAL_FILENAME, fileName);
+ }
+ if (!fileName.empty()) {
+ fileName += ".zip";
+ session->putAttribute(flowFile, FlowAttributeKey(FILENAME), fileName);
+ }
+ return flowFile;
+}
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/libarchive/MergeContent.h
----------------------------------------------------------------------
diff --git a/extensions/libarchive/MergeContent.h b/extensions/libarchive/MergeContent.h
new file mode 100644
index 0000000..312790c
--- /dev/null
+++ b/extensions/libarchive/MergeContent.h
@@ -0,0 +1,342 @@
+/**
+ * @file MergeContent.h
+ * MergeContent class declaration
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __MERGE_CONTENT_H__
+#define __MERGE_CONTENT_H__
+
+#include "BinFiles.h"
+#include "archive_entry.h"
+#include "archive.h"
+#include "core/logging/LoggerConfiguration.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace processors {
+
+#define MERGE_STRATEGY_BIN_PACK "Bin-Packing Algorithm"
+#define MERGE_STRATEGY_DEFRAGMENT "Defragment"
+#define MERGE_FORMAT_TAR_VALUE "TAR"
+#define MERGE_FORMAT_ZIP_VALUE "ZIP"
+#define MERGE_FORMAT_FLOWFILE_STREAM_V3_VALUE "FlowFile Stream, v3"
+#define MERGE_FORMAT_FLOWFILE_STREAM_V2_VALUE "FlowFile Stream, v2"
+#define MERGE_FORMAT_FLOWFILE_TAR_V1_VALUE "FlowFile Tar, v1"
+#define MERGE_FORMAT_CONCAT_VALUE "Binary Concatenation"
+#define MERGE_FORMAT_AVRO_VALUE "Avro"
+#define DELIMITER_STRATEGY_FILENAME "Filename"
+#define DELIMITER_STRATEGY_TEXT "Text"
+
+// MergeBin Class
+class MergeBin {
+public:
+ virtual std::string getMergedContentType() = 0;
+ // merge the flows in the bin
+ virtual std::shared_ptr<core::FlowFile> merge(core::ProcessContext *context, core::ProcessSession *session,
+ std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header, std::string &footer, std::string &demarcator) = 0;
+};
+
+// BinaryConcatenationMerge Class
+class BinaryConcatenationMerge : public MergeBin {
+public:
+ static const char *mimeType;
+ std::string getMergedContentType() {
+ return mimeType;
+ }
+ std::shared_ptr<core::FlowFile> merge(core::ProcessContext *context, core::ProcessSession *session,
+ std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header, std::string &footer, std::string &demarcator);
+ // Nest Callback Class for read stream
+ class ReadCallback : public InputStreamCallback {
+ public:
+ ReadCallback(uint64_t size, std::shared_ptr<io::BaseStream> stream)
+ : buffer_size_(size), stream_(stream) {
+ }
+ ~ReadCallback() {
+ }
+ int64_t process(std::shared_ptr<io::BaseStream> stream) {
+ int max_read = getpagesize();
+ uint8_t buffer[max_read];
+ int64_t ret = 0;
+ uint64_t read_size = 0;
+ while (read_size < buffer_size_) {
+ int readRet = stream->read(buffer, sizeof(buffer));
+ if (readRet > 0) {
+ ret += stream_->write(buffer, readRet);
+ read_size += readRet;
+ } else {
+ break;
+ }
+ }
+ return ret;
+ }
+ uint64_t buffer_size_;
+ std::shared_ptr<io::BaseStream> stream_;
+ };
+ // Nest Callback Class for write stream
+ class WriteCallback: public OutputStreamCallback {
+ public:
+ WriteCallback(std::string &header, std::string &footer, std::string &demarcator, std::deque<std::shared_ptr<core::FlowFile>> &flows, core::ProcessSession *session) :
+ header_(header), footer_(footer), demarcator_(demarcator), flows_(flows), session_(session) {
+ }
+ std::string &header_;
+ std::string &footer_;
+ std::string &demarcator_;
+ std::deque<std::shared_ptr<core::FlowFile>> &flows_;
+ core::ProcessSession *session_;
+ int64_t process(std::shared_ptr<io::BaseStream> stream) {
+ int64_t ret = 0;
+ if (!header_.empty()) {
+ int64_t len = stream->write(reinterpret_cast<uint8_t*>(const_cast<char*>(header_.data())), header_.size());
+ if (len < 0)
+ return len;
+ ret += len;
+ }
+ bool isFirst = true;
+ for (auto flow : flows_) {
+ if (!isFirst && !demarcator_.empty()) {
+ int64_t len = stream->write(reinterpret_cast<uint8_t*>(const_cast<char*>(demarcator_.data())), demarcator_.size());
+ if (len < 0)
+ return len;
+ ret += len;
+ }
+ ReadCallback readCb(flow->getSize(), stream);
+ session_->read(flow, &readCb);
+ ret += flow->getSize();
+ isFirst = false;
+ }
+ if (!footer_.empty()) {
+ int64_t len = stream->write(reinterpret_cast<uint8_t*>(const_cast<char*>(footer_.data())), footer_.size());
+ if (len < 0)
+ return len;
+ ret += len;
+ }
+ return ret;
+ }
+ };
+};
+
+
+// Archive Class
+class ArchiveMerge {
+public:
+ // Nest Callback Class for read stream
+ class ReadCallback: public InputStreamCallback {
+ public:
+ ReadCallback(uint64_t size, struct archive *arch, struct archive_entry *entry) :
+ buffer_size_(size), arch_(arch), entry_(entry) {
+ }
+ ~ReadCallback() {
+ }
+ int64_t process(std::shared_ptr<io::BaseStream> stream) {
+ int max_read = getpagesize();
+ uint8_t buffer[max_read];
+ int64_t ret = 0;
+ uint64_t read_size = 0;
+ ret = archive_write_header(arch_, entry_);
+ while (read_size < buffer_size_) {
+ int readRet = stream->read(buffer, sizeof(buffer));
+ if (readRet > 0) {
+ ret += archive_write_data(arch_, buffer, readRet);
+ read_size += readRet;
+ }
+ else {
+ break;
+ }
+ }
+ return ret;
+ }
+ uint64_t buffer_size_;
+ struct archive *arch_;
+ struct archive_entry *entry_;
+ };
+ // Nest Callback Class for write stream
+ class WriteCallback: public OutputStreamCallback {
+ public:
+ WriteCallback(std::string merge_type, std::deque<std::shared_ptr<core::FlowFile>> &flows, core::ProcessSession *session) :
+ merge_type_(merge_type), flows_(flows), session_(session),
+ logger_(logging::LoggerFactory<ArchiveMerge>::getLogger()) {
+ size_ = 0;
+ stream_ = nullptr;
+ }
+ ~WriteCallback() {
+ }
+
+ std::string merge_type_;
+ std::deque<std::shared_ptr<core::FlowFile>> &flows_;
+ core::ProcessSession *session_;
+ std::shared_ptr<io::BaseStream> stream_;
+ int64_t size_;
+ std::shared_ptr<logging::Logger> logger_;
+
+ static la_ssize_t archive_write(struct archive *arch, void *context, const void *buff, size_t size) {
+ WriteCallback *callback = (WriteCallback *) context;
+ la_ssize_t ret = callback->stream_->write(reinterpret_cast<uint8_t*>(const_cast<void*>(buff)), size);
+ if (ret > 0)
+ callback->size_ += (int64_t) ret;
+ return ret;
+ }
+
+ int64_t process(std::shared_ptr<io::BaseStream> stream) {
+ int64_t ret = 0;
+ struct archive *arch;
+
+ arch = archive_write_new();
+ if (merge_type_ == MERGE_FORMAT_TAR_VALUE) {
+ archive_write_set_format_pax_restricted(arch); // tar format
+ }
+ if (merge_type_ == MERGE_FORMAT_ZIP_VALUE) {
+ archive_write_set_format_zip(arch); // zip format
+ }
+ archive_write_set_bytes_per_block(arch, 0);
+ archive_write_add_filter_none(arch);
+ this->stream_ = stream;
+ archive_write_open(arch, this, NULL, archive_write, NULL);
+
+ for (auto flow : flows_) {
+ struct archive_entry *entry = archive_entry_new();
+ std::string fileName;
+ flow->getAttribute(FlowAttributeKey(FILENAME), fileName);
+ archive_entry_set_pathname(entry, fileName.c_str());
+ archive_entry_set_size(entry, flow->getSize());
+ archive_entry_set_mode(entry, S_IFREG | 0755);
+ if (merge_type_ == MERGE_FORMAT_TAR_VALUE) {
+ std::string perm;
+ int permInt;
+ if (flow->getAttribute(BinFiles::TAR_PERMISSIONS_ATTRIBUTE, perm)) {
+ try {
+ permInt = std::stoi(perm);
+ logger_->log_info("Merge Tar File %s permission %s", fileName, perm);
+ archive_entry_set_perm(entry, (mode_t) permInt);
+ } catch (...) {
+ }
+ }
+ }
+ ReadCallback readCb(flow->getSize(), arch, entry);
+ session_->read(flow, &readCb);
+ archive_entry_free(entry);
+ }
+
+ archive_write_close(arch);
+ archive_write_free(arch);
+ return size_;
+ }
+ };
+};
+
+// TarMerge Class
+class TarMerge: public ArchiveMerge, public MergeBin {
+public:
+ static const char *mimeType;
+ std::shared_ptr<core::FlowFile> merge(core::ProcessContext *context, core::ProcessSession *session, std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header, std::string &footer,
+ std::string &demarcator);
+ std::string getMergedContentType() {
+ return mimeType;
+ }
+};
+
+// ZipMerge Class
+class ZipMerge: public ArchiveMerge, public MergeBin {
+public:
+ static const char *mimeType;
+ std::shared_ptr<core::FlowFile> merge(core::ProcessContext *context, core::ProcessSession *session, std::deque<std::shared_ptr<core::FlowFile>> &flows, std::string &header, std::string &footer,
+ std::string &demarcator);
+ std::string getMergedContentType() {
+ return mimeType;
+ }
+};
+
+// MergeContent Class
+class MergeContent : public processors::BinFiles {
+ public:
+ // Constructor
+ /*!
+ * Create a new processor
+ */
+ explicit MergeContent(std::string name, uuid_t uuid = NULL)
+ : processors::BinFiles(name, uuid),
+ logger_(logging::LoggerFactory<MergeContent>::getLogger()) {
+ mergeStratgey_ = MERGE_STRATEGY_DEFRAGMENT;
+ mergeFormat_ = MERGE_FORMAT_CONCAT_VALUE;
+ delimiterStratgey_ = DELIMITER_STRATEGY_FILENAME;
+ keepPath_ = false;
+ }
+ // Destructor
+ virtual ~MergeContent() {
+ }
+ // Processor Name
+ static constexpr char const* ProcessorName = "MergeContent";
+ // Supported Properties
+ static core::Property MergeStrategy;
+ static core::Property MergeFormat;
+ static core::Property CorrelationAttributeName;
+ static core::Property DelimiterStratgey;
+ static core::Property KeepPath;
+ static core::Property Header;
+ static core::Property Footer;
+ static core::Property Demarcator;
+
+ // Supported Relationships
+ static core::Relationship Merge;
+
+ public:
+ /**
+ * Function that's executed when the processor is scheduled.
+ * @param context process context.
+ * @param sessionFactory process session factory that is used when creating
+ * ProcessSession objects.
+ */
+ void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory);
+ // OnTrigger method, implemented by NiFi MergeContent
+ virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session);
+ // Initialize, over write by NiFi MergeContent
+ virtual void initialize(void);
+ virtual bool processBin(core::ProcessContext *context, core::ProcessSession *session, std::unique_ptr<Bin> &bin);
+
+ protected:
+ // Returns a group ID representing a bin. This allows flow files to be binned into like groups
+ virtual std::string getGroupId(core::ProcessContext *context, std::shared_ptr<core::FlowFile> flow);
+ // check whether the defragment bin is validate
+ bool checkDefragment(std::unique_ptr<Bin> &bin);
+
+ private:
+ std::shared_ptr<logging::Logger> logger_;
+ std::string mergeStratgey_;
+ std::string mergeFormat_;
+ std::string correlationAttributeName_;
+ bool keepPath_;
+ std::string delimiterStratgey_;
+ std::string header_;
+ std::string footer_;
+ std::string demarcator_;
+ std::string headerContent_;
+ std::string footerContent_;
+ std::string demarcatorContent_;
+ // readContent
+ std::string readContent(std::string path);
+};
+
+REGISTER_RESOURCE(MergeContent);
+
+} /* namespace processors */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/rocksdb-repos/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt
index 5edbfb3..76dd1f2 100644
--- a/extensions/rocksdb-repos/CMakeLists.txt
+++ b/extensions/rocksdb-repos/CMakeLists.txt
@@ -19,18 +19,14 @@
cmake_minimum_required(VERSION 2.6)
-find_package(CURL REQUIRED)
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols")
set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols")
-include_directories(../../libminifi/include ../../libminifi/include/c2 ../../libminifi/include/c2/protocols/ ../../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/rocksdb/include ../../thirdparty/libarchive-3.3.2/libarchive ../../thirdparty/)
-
-find_package(Boost REQUIRED)
+include_directories(../../libminifi/include ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/rocksdb/include ../../thirdparty/)
find_package(RocksDB)
-include_directories(${Boost_INCLUDE_DIRS})
file(GLOB SOURCES "*.cpp")
@@ -43,10 +39,6 @@ if(CMAKE_THREAD_LIBS_INIT)
target_link_libraries(minifi-rocksdb-repos "${CMAKE_THREAD_LIBS_INIT}")
endif()
-if (CURL_FOUND)
- include_directories(${CURL_INCLUDE_DIRS})
- target_link_libraries (minifi-rocksdb-repos ${CURL_LIBRARIES})
-endif(CURL_FOUND)
# Include UUID
find_package(UUID REQUIRED)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/extensions/rocksdb-repos/DatabaseContentRepository.h
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.h b/extensions/rocksdb-repos/DatabaseContentRepository.h
index e43ff35..676e9dd 100644
--- a/extensions/rocksdb-repos/DatabaseContentRepository.h
+++ b/extensions/rocksdb-repos/DatabaseContentRepository.h
@@ -22,7 +22,7 @@
#include "rocksdb/merge_operator.h"
#include "core/Core.h"
#include "core/Connectable.h"
-#include "../ContentRepository.h"
+#include "core/ContentRepository.h"
#include "properties/Configure.h"
#include "core/logging/LoggerConfiguration.h"
namespace org {
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/libminifi/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt
index e3a3199..6eea752 100644
--- a/libminifi/CMakeLists.txt
+++ b/libminifi/CMakeLists.txt
@@ -19,19 +19,14 @@
cmake_minimum_required (VERSION 2.6)
-project(nifi-libminifi)
-set(PROJECT_NAME "nifi-libminifi")
+project(nifi-libcore-minifi)
+set(PROJECT_NAME "nifi-libcore-minifi")
set(PROJECT_VERSION_MAJOR 0)
set(PROJECT_VERSION_MINOR 3)
set(PROJECT_VERSION_PATCH 0)
#### Establish Project Configuration ####
# Enable usage of the VERSION specifier
-# https://cmake.org/cmake/help/v3.0/policy/CMP0048.html#policy:CMP0048
-IF(POLICY CMP0048)
- CMAKE_POLICY(SET CMP0048 OLD)
-ENDIF(POLICY CMP0048)
-
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -56,59 +51,44 @@ else()
endif()
include_directories(../thirdparty/spdlog-20170710/include)
-include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include)
+include_directories(../thirdparty/yaml-cpp-yaml-cpp-20171024/include)
include_directories(../thirdparty/civetweb-1.9.1/include)
include_directories(../thirdparty/jsoncpp/include)
include_directories(../thirdparty/concurrentqueue/)
-include_directories(../thirdparty/rocksdb/include)
-include_directories(../thirdparty/libarchive-3.3.2/libarchive)
include_directories(include)
-file(GLOB SOURCES "src/core/logging/*.cpp" "src/core/state/*.cpp" "src/c2/protocols/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/utils/*.cpp" "src/*.cpp")
+file(GLOB SOURCES "src/core/logging/*.cpp" "src/core/state/*.cpp" "src/c2/protocols/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/utils/*.cpp" "src/*.cpp")
+
+file(GLOB PROCESSOR_SOURCES "src/processors/*.cpp" )
file(GLOB SPD_SOURCES "../thirdparty/spdlog-20170710/include/spdlog/*")
# Workaround the limitations of having a
# header only library
add_library(spdlog STATIC ${SPD_SOURCES})
-add_library(minifi STATIC ${SOURCES})
-
-add_dependencies(minifi jsoncpp_project)
-target_link_libraries(minifi ${UUID_LIBRARIES} ${JSONCPP_LIB})
+add_library(core-minifi STATIC ${SOURCES})
+add_dependencies(core-minifi jsoncpp_project)
+target_link_libraries(core-minifi ${UUID_LIBRARIES} ${JSONCPP_LIB} yaml-cpp )
find_package(ZLIB REQUIRED)
include_directories(${ZLIB_INCLUDE_DIRS})
-target_link_libraries (minifi ${ZLIB_LIBRARIES})
-
-
-
-
-
-if (NOT IOS)
-# Include Boost System
-find_package(Boost COMPONENTS system filesystem REQUIRED)
-target_link_libraries(minifi ${Boost_SYSTEM_LIBRARY})
-target_link_libraries(minifi ${Boost_FILESYSTEM_LIBRARY})
-
-if (CURL_FOUND)
- include_directories(${CURL_INCLUDE_DIRS})
- target_link_libraries (minifi ${CURL_LIBRARIES})
-endif(CURL_FOUND)
+target_link_libraries (core-minifi ${ZLIB_LIBRARIES})
# Include OpenSSL
find_package (OpenSSL REQUIRED)
if (OPENSSL_FOUND)
include_directories(${OPENSSL_INCLUDE_DIR})
- target_link_libraries (minifi ${OPENSSL_LIBRARIES})
+ target_link_libraries (core-minifi ${OPENSSL_LIBRARIES})
else ()
message( FATAL_ERROR "OpenSSL was not found. Please install OpenSSL" )
endif (OPENSSL_FOUND)
-endif ()
+add_library(minifi STATIC ${PROCESSOR_SOURCES})
+target_link_libraries(minifi core-minifi)
set_property(TARGET minifi PROPERTY INTERPROCEDURAL_OPTIMIZATION True)
-SET (LIBMINIFI minifi PARENT_SCOPE)
+SET (LIBMINIFI core-minifi PARENT_SCOPE)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/1d6b2416/libminifi/include/core/FlowConfiguration.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h
index 6c50772..023e100 100644
--- a/libminifi/include/core/FlowConfiguration.h
+++ b/libminifi/include/core/FlowConfiguration.h
@@ -34,7 +34,6 @@
#include "processors/LogAttribute.h"
#include "processors/ExecuteProcess.h"
#include "processors/AppendHostInfo.h"
-#include "processors/MergeContent.h"
#include "core/Processor.h"
#include "core/logging/LoggerConfiguration.h"
#include "core/ProcessContext.h"