You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2021/06/23 16:55:51 UTC
[nifi-minifi-cpp] 03/06: MINIFICPP-1580 Fix Tensorflow extension
build
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit e6ecb2e182008e3d6f7bdebc591409fafd2cd6c9
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Wed Jun 23 18:14:24 2021 +0200
MINIFICPP-1580 Fix Tensorflow extension build
Closes #1096
Signed-off-by: Marton Szasz <sz...@apache.org>
---
cmake/FindTensorFlow.cmake | 55 ++++++++++++++++------
extensions/tensorflow/CMakeLists.txt | 3 +-
extensions/tensorflow/TFApplyGraph.cpp | 11 +++--
extensions/tensorflow/TFApplyGraph.h | 2 +-
extensions/tensorflow/TFConvertImageToTensor.cpp | 6 +--
extensions/tensorflow/TFConvertImageToTensor.h | 2 +-
extensions/tensorflow/TFExtractTopLabels.cpp | 11 +++--
extensions/tensorflow/TFExtractTopLabels.h | 2 +-
libminifi/test/tensorflow-tests/CMakeLists.txt | 2 +-
.../test/tensorflow-tests/TensorFlowTests.cpp | 10 ++--
10 files changed, 67 insertions(+), 37 deletions(-)
diff --git a/cmake/FindTensorFlow.cmake b/cmake/FindTensorFlow.cmake
index 732e3ae..2996048 100644
--- a/cmake/FindTensorFlow.cmake
+++ b/cmake/FindTensorFlow.cmake
@@ -18,8 +18,20 @@
include(FindPackageHandleStandardArgs)
unset(TENSORFLOW_FOUND)
-if (TENSORFLOW_PATH)
- message("-- Checking for TensorFlow in provided TENSORFLOW_PATH: ${TENSORFLOW_PATH}")
+if (TENSORFLOW_INCLUDE_PATH)
+ message("-- Checking for TensorFlow includes in provided TENSORFLOW_INCLUDE_PATH: ${TENSORFLOW_INCLUDE_PATH}")
+endif()
+
+if (TENSORFLOW_LIB_PATH)
+ message("-- Checking for TensorFlow libs in provided TENSORFLOW_LIB_PATH: ${TENSORFLOW_LIB_PATH}")
+endif()
+
+if (GOOGLE_PROTOBUF_INCLUDE_PATH)
+ message("-- Checking for Google Protobuf includes in provided GOOGLE_PROTOBUF_INCLUDE_PATH: ${GOOGLE_PROTOBUF_INCLUDE_PATH}")
+endif()
+
+if (GOOGLE_PROTOBUF_LIB_PATH)
+ message("-- Checking for Google Protobuf libs in provided GOOGLE_PROTOBUF_LIB_PATH: ${GOOGLE_PROTOBUF_LIB_PATH}")
endif()
find_path(TENSORFLOW_INCLUDE_DIR
@@ -28,34 +40,46 @@ find_path(TENSORFLOW_INCLUDE_DIR
tensorflow/cc
third_party
HINTS
- ${TENSORFLOW_PATH}
+ ${TENSORFLOW_INCLUDE_PATH}
/usr/include/tensorflow
/usr/local/include/google/tensorflow
/usr/local/include/tensorflow
+ /usr/local/include/tensorflow/bazel-bin/tensorflow/include
/usr/include/google/tensorflow)
find_library(TENSORFLOW_CC_LIBRARY NAMES tensorflow_cc
HINTS
- ${TENSORFLOW_PATH}
- ${TENSORFLOW_PATH}/bazel-bin/tensorflow
+ ${TENSORFLOW_LIB_PATH}
+ ${TENSORFLOW_INCLUDE_PATH}/bazel-bin/tensorflow
/usr/lib
/usr/local/lib
/usr/local/lib/tensorflow_cc)
-find_library(TENSORFLOW_FRAMEWORK_LIBRARY NAMES tensorflow_framework
+find_path(GOOGLE_PROTOBUF_INCLUDE NAMES google/protobuf
+ HINTS
+ ${GOOGLE_PROTOBUF_INCLUDE_PATH}
+ ${TENSORFLOW_INCLUDE_PATH}/src
+ /usr/include/tensorflow/src
+ /usr/local/include/google/tensorflow/src
+ /usr/local/include/tensorflow/src
+ /usr/local/include/tensorflow/bazel-bin/tensorflow/include/src
+ /usr/include/google/tensorflow/src)
+
+find_library(GOOGLE_PROTOBUF_LIBRARY NAMES protobuf
HINTS
- ${TENSORFLOW_PATH}
- ${TENSORFLOW_PATH}/bazel-bin/tensorflow
+ ${GOOGLE_PROTOBUF_LIB_PATH}
/usr/lib
/usr/local/lib
- /usr/local/lib/tensorflow_cc)
+ /usr/lib/x86_64-linux-gnu)
-find_package_handle_standard_args(TENSORFLOW DEFAULT_MSG TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY TENSORFLOW_FRAMEWORK_LIBRARY)
+find_package_handle_standard_args(TENSORFLOW DEFAULT_MSG TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY GOOGLE_PROTOBUF_INCLUDE GOOGLE_PROTOBUF_LIBRARY)
if(TENSORFLOW_FOUND)
message("-- Found TensorFlow includes: ${TENSORFLOW_INCLUDE_DIR}")
- message("-- Found TensorFlow libraries: ${TENSORFLOW_CC_LIBRARY} ${TENSORFLOW_FRAMEWORK_LIBRARY}")
- set(TENSORFLOW_LIBRARIES ${TENSORFLOW_CC_LIBRARY} ${TENSORFLOW_FRAMEWORK_LIBRARY})
+ message("-- Found TensorFlow libs: ${TENSORFLOW_CC_LIBRARY}")
+ message("-- Found Google Protobuf includes: ${GOOGLE_PROTOBUF_INCLUDE}")
+ message("-- Found Google Protobuf libs: ${GOOGLE_PROTOBUF_LIBRARY}")
+ set(TENSORFLOW_LIBRARIES ${TENSORFLOW_CC_LIBRARY} ${GOOGLE_PROTOBUF_LIBRARY})
set(TENSORFLOW_INCLUDE_DIRS
${TENSORFLOW_INCLUDE_DIR}
${TENSORFLOW_INCLUDE_DIR}/bazel-genfiles
@@ -63,9 +87,10 @@ if(TENSORFLOW_FOUND)
${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/eigen
${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/gemmlowp
${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/nsync/public
- ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/gen/protobuf-host/include)
+ ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/gen/protobuf-host/include
+ ${GOOGLE_PROTOBUF_INCLUDE})
else()
- message(FATAL_ERROR "TensorFlow was not found. Check or set TENSORFLOW_PATH to TensorFlow build, Install libtensorflow_cc.so and headers into the system, or disable the TensorFlow extension.")
+ message(FATAL_ERROR "TensorFlow or Google Protobuf dependency was not found. Check or set TENSORFLOW_INCLUDE_PATH, TENSORFLOW_LIB_PATH, GOOGLE_PROTOBUF_INCLUDE, GOOGLE_PROTOBUF_LIBRARY to TensorFlow build, Install libtensorflow_cc.so, libprotobuf.so and headers into the system, or disable the TensorFlow extension.")
endif()
-mark_as_advanced(TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY TENSORFLOW_FRAMEWORK_LIBRARY)
+mark_as_advanced(TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY GOOGLE_PROTOBUF_INCLUDE GOOGLE_PROTOBUF_LIBRARY)
diff --git a/extensions/tensorflow/CMakeLists.txt b/extensions/tensorflow/CMakeLists.txt
index 3c792ee..b6f74c2 100644
--- a/extensions/tensorflow/CMakeLists.txt
+++ b/extensions/tensorflow/CMakeLists.txt
@@ -26,13 +26,12 @@ find_package(TensorFlow REQUIRED)
message("-- Found TensorFlow: ${TENSORFLOW_INCLUDE_DIRS}")
-include_directories(${TENSORFLOW_INCLUDE_DIRS})
-
file(GLOB SOURCES "*.cpp")
add_library(minifi-tensorflow-extensions STATIC ${SOURCES})
set_property(TARGET minifi-tensorflow-extensions PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_include_directories(minifi-tensorflow-extensions SYSTEM PRIVATE ${TENSORFLOW_INCLUDE_DIRS})
target_link_libraries(minifi-tensorflow-extensions ${LIBMINIFI} Threads::Threads)
target_link_libraries(minifi-tensorflow-extensions ${TENSORFLOW_LIBRARIES})
diff --git a/extensions/tensorflow/TFApplyGraph.cpp b/extensions/tensorflow/TFApplyGraph.cpp
index 14d349e..b14c2e3 100644
--- a/extensions/tensorflow/TFApplyGraph.cpp
+++ b/extensions/tensorflow/TFApplyGraph.cpp
@@ -65,7 +65,7 @@ void TFApplyGraph::initialize() {
setSupportedRelationships(std::move(relationships));
}
-void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) {
+void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory* /*sessionFactory*/) {
context->getProperty(InputNode.getName(), input_node_);
if (input_node_.empty()) {
@@ -79,8 +79,8 @@ void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessio
}
}
-void TFApplyGraph::onTrigger(const std::shared_ptr<core::ProcessContext> &context,
- const std::shared_ptr<core::ProcessSession> &session) {
+void TFApplyGraph::onTrigger(const std::shared_ptr<core::ProcessContext>& /*context*/,
+ const std::shared_ptr<core::ProcessSession>& session) {
auto flow_file = session->get();
if (!flow_file) {
@@ -152,7 +152,10 @@ void TFApplyGraph::onTrigger(const std::shared_ptr<core::ProcessContext> &contex
TensorReadCallback tensor_cb(input_tensor_proto);
session->read(flow_file, &tensor_cb);
tensorflow::Tensor input;
- input.FromProto(*input_tensor_proto);
+ if (!input.FromProto(*input_tensor_proto)) {
+ // failure deliberately ignored at this time
+ // added to avoid warn_unused_result build errors
+ }
std::vector<tensorflow::Tensor> outputs;
auto status = ctx->tf_session->Run({{input_node_, input}}, {output_node_}, {}, &outputs);
diff --git a/extensions/tensorflow/TFApplyGraph.h b/extensions/tensorflow/TFApplyGraph.h
index 3f26872..e00d528 100644
--- a/extensions/tensorflow/TFApplyGraph.h
+++ b/extensions/tensorflow/TFApplyGraph.h
@@ -47,7 +47,7 @@ class TFApplyGraph : public core::Processor {
void initialize() override;
void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override;
- void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+ void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override {
logger_->log_error("onTrigger invocation with raw pointers is not implemented");
}
void onTrigger(const std::shared_ptr<core::ProcessContext> &context,
diff --git a/extensions/tensorflow/TFConvertImageToTensor.cpp b/extensions/tensorflow/TFConvertImageToTensor.cpp
index aea09f8..efa4716 100644
--- a/extensions/tensorflow/TFConvertImageToTensor.cpp
+++ b/extensions/tensorflow/TFConvertImageToTensor.cpp
@@ -115,7 +115,7 @@ void TFConvertImageToTensor::initialize() {
setSupportedRelationships(std::move(relationships));
}
-void TFConvertImageToTensor::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) {
+void TFConvertImageToTensor::onSchedule(core::ProcessContext* context, core::ProcessSessionFactory* /*sessionFactory*/) {
context->getProperty(ImageFormat.getName(), input_format_);
if (input_format_.empty()) {
@@ -189,8 +189,8 @@ void TFConvertImageToTensor::onSchedule(core::ProcessContext *context, core::Pro
}
}
-void TFConvertImageToTensor::onTrigger(const std::shared_ptr<core::ProcessContext> &context,
- const std::shared_ptr<core::ProcessSession> &session) {
+void TFConvertImageToTensor::onTrigger(const std::shared_ptr<core::ProcessContext>& /*context*/,
+ const std::shared_ptr<core::ProcessSession>& session) {
auto flow_file = session->get();
if (!flow_file) {
diff --git a/extensions/tensorflow/TFConvertImageToTensor.h b/extensions/tensorflow/TFConvertImageToTensor.h
index da3a11c..93c5b38 100644
--- a/extensions/tensorflow/TFConvertImageToTensor.h
+++ b/extensions/tensorflow/TFConvertImageToTensor.h
@@ -54,7 +54,7 @@ class TFConvertImageToTensor : public core::Processor {
void initialize() override;
void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override;
- void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+ void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override {
logger_->log_error("onTrigger invocation with raw pointers is not implemented");
}
void onTrigger(const std::shared_ptr<core::ProcessContext> &context,
diff --git a/extensions/tensorflow/TFExtractTopLabels.cpp b/extensions/tensorflow/TFExtractTopLabels.cpp
index 2f44f84..cfdaa9d 100644
--- a/extensions/tensorflow/TFExtractTopLabels.cpp
+++ b/extensions/tensorflow/TFExtractTopLabels.cpp
@@ -48,10 +48,10 @@ void TFExtractTopLabels::initialize() {
setSupportedRelationships(std::move(relationships));
}
-void TFExtractTopLabels::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) {
+void TFExtractTopLabels::onSchedule(core::ProcessContext* /*context*/, core::ProcessSessionFactory* /*sessionFactory*/) {
}
-void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext> &context,
+void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext>& /*context*/,
const std::shared_ptr<core::ProcessSession> &session) {
auto flow_file = session->get();
@@ -88,7 +88,10 @@ void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext> &
session->read(flow_file, &tensor_cb);
tensorflow::Tensor input;
- input.FromProto(*input_tensor_proto);
+ if (!input.FromProto(*input_tensor_proto)) {
+ // failure deliberately ignored at this time
+ // added to avoid warn_unused_result build errors
+ }
auto input_flat = input.flat<float>();
std::vector<std::pair<uint64_t, float>> scores;
@@ -102,7 +105,7 @@ void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext> &
return a.second > b.second;
});
- for (int i = 0; i < 5 && i < scores.size(); i++) {
+ for (std::size_t i = 0; i < 5 && i < scores.size(); i++) {
if (!labels || scores[i].first > labels->size()) {
logger_->log_error("Label index is out of range (are the correct labels loaded?); routing to retry...");
session->transfer(flow_file, Retry);
diff --git a/extensions/tensorflow/TFExtractTopLabels.h b/extensions/tensorflow/TFExtractTopLabels.h
index a1b2211..03ca8ca 100644
--- a/extensions/tensorflow/TFExtractTopLabels.h
+++ b/extensions/tensorflow/TFExtractTopLabels.h
@@ -44,7 +44,7 @@ class TFExtractTopLabels : public core::Processor {
void initialize() override;
void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override;
- void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+ void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override {
logger_->log_error("onTrigger invocation with raw pointers is not implemented");
}
void onTrigger(const std::shared_ptr<core::ProcessContext> &context,
diff --git a/libminifi/test/tensorflow-tests/CMakeLists.txt b/libminifi/test/tensorflow-tests/CMakeLists.txt
index c7e68eb..97b45ec 100644
--- a/libminifi/test/tensorflow-tests/CMakeLists.txt
+++ b/libminifi/test/tensorflow-tests/CMakeLists.txt
@@ -27,7 +27,7 @@ FOREACH(testfile ${TENSORFLOW_INTEGRATION_TESTS})
add_executable("${testfilename}" "${testfile}" "${TEST_DIR}/TestBase.cpp")
target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/tensorflow")
target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors")
- target_include_directories(${testfilename} PRIVATE BEFORE ${TENSORFLOW_INCLUDE_DIRS})
+ target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE ${TENSORFLOW_INCLUDE_DIRS})
createTests("${testfilename}")
target_wholearchive_library(${testfilename} minifi-tensorflow-extensions)
target_wholearchive_library(${testfilename} minifi-standard-processors)
diff --git a/libminifi/test/tensorflow-tests/TensorFlowTests.cpp b/libminifi/test/tensorflow-tests/TensorFlowTests.cpp
index d4dc27c..6959cfb 100644
--- a/libminifi/test/tensorflow-tests/TensorFlowTests.cpp
+++ b/libminifi/test/tensorflow-tests/TensorFlowTests.cpp
@@ -120,13 +120,13 @@ TEST_CASE("TensorFlow: Apply Graph", "[tfApplyGraph]") { // NOLINT
tensorflow::GraphDef graph;
// Write test TensorFlow graph
- root.ToGraphDef(&graph);
+ REQUIRE(root.ToGraphDef(&graph).ok());
std::ofstream in_file_stream(in_graph_file);
graph.SerializeToOstream(&in_file_stream);
}
// Read test TensorFlow graph into TFApplyGraph
- plan->runNextProcessor([&get_file, &in_graph_file, &plan](const std::shared_ptr<core::ProcessContext> context,
+ plan->runNextProcessor([&in_graph_file](const std::shared_ptr<core::ProcessContext> /*context*/,
const std::shared_ptr<core::ProcessSession> session) {
// Intercept the call so that we can add an attr (won't be required when we have UpdateAttribute processor)
auto flow_file = session->create();
@@ -163,7 +163,7 @@ TEST_CASE("TensorFlow: Apply Graph", "[tfApplyGraph]") { // NOLINT
tensorflow::TensorProto tensor_proto;
tensor_proto.ParseFromIstream(&out_file_stream);
tensorflow::Tensor tensor;
- tensor.FromProto(tensor_proto);
+ REQUIRE(tensor.FromProto(tensor_proto));
// Verify output tensor
float tensor_val = tensor.flat<float>().data()[0];
@@ -286,7 +286,7 @@ TEST_CASE("TensorFlow: ConvertImageToTensor", "[tfConvertImageToTensor]") { // N
tensorflow::TensorProto tensor_proto;
tensor_proto.ParseFromIstream(&out_file_stream);
tensorflow::Tensor tensor;
- tensor.FromProto(tensor_proto);
+ REQUIRE(tensor.FromProto(tensor_proto));
// Verify output tensor
auto shape = tensor.shape();
@@ -358,7 +358,7 @@ TEST_CASE("TensorFlow: Extract Top Labels", "[tfExtractTopLabels]") { // NOLINT
}
// Read labels
- plan->runNextProcessor([&get_file, &in_labels_file, &plan](const std::shared_ptr<core::ProcessContext> context,
+ plan->runNextProcessor([&in_labels_file](const std::shared_ptr<core::ProcessContext> /*context*/,
const std::shared_ptr<core::ProcessSession> session) {
// Intercept the call so that we can add an attr (won't be required when we have UpdateAttribute processor)
auto flow_file = session->create();