You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by fg...@apache.org on 2022/06/01 11:21:11 UTC
[nifi-minifi-cpp] 02/03: MINIFICPP-1461 Enable Expression Languaage tests on Windows
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit b41bed42305455e5006ce855cec86d47df344d0a
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu Apr 14 09:32:07 2022 +0200
MINIFICPP-1461 Enable Expression Languaage tests on Windows
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
This closes #1337
---
CMakeLists.txt | 5 +-
cmake/Date.cmake | 14 +++--
extensions/expression-language/Expression.cpp | 68 ++--------------------
extensions/expression-language/common/Value.h | 4 +-
.../impl/expression/Expression.h | 13 ++---
.../expression-language/tests/CMakeLists.txt | 43 +++++++-------
.../tests/ExpressionLanguageTests.cpp | 34 +++++------
.../tests/ProcessContextExprTests.cpp | 4 +-
8 files changed, 59 insertions(+), 126 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 871e061d3..a0830c18d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -646,10 +646,7 @@ set(CPACK_COMPONENTS_GROUPING "ALL_COMPONENTS_IN_ONE")
list(APPEND CPACK_COMPONENTS_ALL bin)
cpack_add_component(bin DISPLAY_NAME "MiNiFi C++ executables" REQUIRED)
-if(WIN32)
- list(APPEND CPACK_COMPONENTS_ALL tzdata)
- cpack_add_component(tzdata DISPLAY_NAME "Timezone database for Expression Language")
-else()
+if(NOT WIN32)
list(APPEND CPACK_COMPONENTS_ALL conf)
cpack_add_component(conf DISPLAY_NAME "Default configuration files" REQUIRED)
endif()
diff --git a/cmake/Date.cmake b/cmake/Date.cmake
index fdec3d1f7..9b6e24948 100644
--- a/cmake/Date.cmake
+++ b/cmake/Date.cmake
@@ -28,15 +28,21 @@ if (WIN32)
FetchContent_Populate(tzdata)
endif()
+ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/tzdata)
+
+ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cldr-common-38.1/common/supplemental/windowsZones.xml
+ DESTINATION ${CMAKE_BINARY_DIR}/tzdata)
+
+ file(COPY ${tzdata_SOURCE_DIR}/
+ DESTINATION ${CMAKE_BINARY_DIR}/tzdata)
+
install(DIRECTORY ${tzdata_SOURCE_DIR}/
DESTINATION tzdata
- COMPONENT tzdata
- )
+ COMPONENT bin)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cldr-common-38.1/common/supplemental/windowsZones.xml
DESTINATION tzdata
- COMPONENT tzdata
- )
+ COMPONENT bin)
endif()
FetchContent_Declare(date_src
diff --git a/extensions/expression-language/Expression.cpp b/extensions/expression-language/Expression.cpp
index fbce1b104..09a42c8f2 100644
--- a/extensions/expression-language/Expression.cpp
+++ b/extensions/expression-language/Expression.cpp
@@ -63,11 +63,7 @@
#include "Driver.h"
-#ifdef EXPRESSION_LANGUAGE_USE_DATE
#include "date/tz.h"
-#else
-#include <ctime>
-#endif // EXPRESSION_LANGUAGE_USE_DATE
namespace org {
namespace apache {
@@ -610,8 +606,6 @@ Value expr_escapeCsv(const std::vector<Value> &args) {
return Value(result);
}
-#ifdef EXPRESSION_LANGUAGE_USE_DATE
-
Value expr_format(const std::vector<Value> &args) {
std::chrono::milliseconds dur(args[0].asUnsignedLong());
std::chrono::system_clock::time_point dt(dur);
@@ -640,52 +634,6 @@ Value expr_toDate(const std::vector<Value> &args) {
return Value(int64_t{std::chrono::duration_cast<std::chrono::milliseconds>(zt.get_sys_time().time_since_epoch()).count()});
}
-#else
-
-Value expr_format(const std::vector<Value>& args) {
- const std::chrono::milliseconds dur(args.at(0).asUnsignedLong());
- const std::chrono::system_clock::time_point dt(dur);
- const auto unix_time = std::chrono::system_clock::to_time_t(dt);
- const auto zoned_time = [&args, unix_time] {
- std::tm buf{};
- const auto requested_timezone = args.size() > 2 ? args[2].asString() : std::string{};
- if (requested_timezone == "UTC" || requested_timezone == "GMT") {
-#ifdef WIN32
- const auto err = gmtime_s(&buf, &unix_time);
- if (!err) { return buf; }
- throw std::system_error{err, std::generic_category()};
-#else
- tzset();
- const std::tm* const result = gmtime_r(&unix_time, &buf);
- if (result) { return *result; }
- throw std::system_error{errno, std::generic_category()};
-#endif /* WIN32 */
- } else if (!requested_timezone.empty()) {
- throw std::domain_error{"format() with Non-UTC custom timezone is only supported when compiled with the date.h library"};
- } else {
-#ifdef WIN32
- const auto err = localtime_s(&buf, &unix_time);
- if (!err) { return buf; }
- throw std::system_error{err, std::generic_category()};
-#else
- tzset();
- const std::tm* const result = localtime_r(&unix_time, &buf);
- if (result) { return *result; }
- throw std::system_error{errno, std::generic_category()};
-#endif /* WIN32 */
- }
- }();
- char result_buf[512] = {0};
- std::strftime(result_buf, 512, args.at(1).asString().c_str(), &zoned_time);
- return Value(std::string(result_buf));
-}
-
-Value expr_toDate(const std::vector<Value>&) {
- throw std::domain_error{"toDate() is only supported when compiled with the date.h library"};
-}
-
-#endif // EXPRESSION_LANGUAGE_USE_DATE
-
Value expr_now(const std::vector<Value>& /*args*/) {
return Value(int64_t{std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count()});
}
@@ -774,8 +722,6 @@ Value expr_base64Decode(const std::vector<Value> &args) {
return Value(utils::StringUtils::from_base64(args[0].asString(), utils::as_string));
}
-#ifdef EXPRESSION_LANGUAGE_USE_REGEX
-
Value expr_replace(const std::vector<Value> &args) {
std::string result = args[0].asString();
const std::string &find = args[1].asString();
@@ -835,8 +781,6 @@ Value expr_find(const std::vector<Value> &args) {
return Value(utils::regexSearch(subject, expr));
}
-#endif // EXPRESSION_LANGUAGE_USE_REGEX
-
Value expr_trim(const std::vector<Value> &args) {
return Value{utils::StringUtils::trim(args[0].asString())};
}
@@ -1160,8 +1104,6 @@ Expression make_anyAttribute(const std::string &function_name, const std::vector
return result;
}
-#ifdef EXPRESSION_LANGUAGE_USE_REGEX
-
Expression make_allMatchingAttributes(const std::string &function_name, const std::vector<Expression> &args) {
if (args.size() < 1) {
std::stringstream message_ss;
@@ -1268,8 +1210,6 @@ Expression make_anyMatchingAttribute(const std::string &function_name, const std
return result;
}
-#endif // EXPRESSION_LANGUAGE_USE_REGEX
-
Expression make_allDelineatedValues(const std::string &function_name, const std::vector<Expression> &args) {
if (args.size() != 2) {
std::stringstream message_ss;
@@ -1457,7 +1397,6 @@ Expression make_dynamic_function(const std::string &function_name, const std::ve
return make_dynamic_function_incomplete<expr_base64Encode>(function_name, args, 0);
} else if (function_name == "base64Decode") {
return make_dynamic_function_incomplete<expr_base64Decode>(function_name, args, 0);
-#ifdef EXPRESSION_LANGUAGE_USE_REGEX
} else if (function_name == "replace") {
return make_dynamic_function_incomplete<expr_replace>(function_name, args, 2);
} else if (function_name == "replaceFirst") {
@@ -1476,7 +1415,6 @@ Expression make_dynamic_function(const std::string &function_name, const std::ve
return make_allMatchingAttributes(function_name, args);
} else if (function_name == "anyMatchingAttribute") {
return make_anyMatchingAttribute(function_name, args);
-#endif // EXPRESSION_LANGUAGE_USE_REGEX
} else if (function_name == "trim") {
return make_dynamic_function_incomplete<expr_trim>(function_name, args, 0);
} else if (function_name == "append") {
@@ -1665,6 +1603,12 @@ Expression Expression::make_aggregate(std::function<Value(const Parameters ¶
});
}
+#ifdef WIN32
+void dateSetInstall(const std::string& install) {
+ date::set_install(install);
+}
+#endif
+
} /* namespace expression */
} /* namespace minifi */
} /* namespace nifi */
diff --git a/extensions/expression-language/common/Value.h b/extensions/expression-language/common/Value.h
index 885775a2d..65d3cf586 100644
--- a/extensions/expression-language/common/Value.h
+++ b/extensions/expression-language/common/Value.h
@@ -180,7 +180,7 @@ class Value {
if (is_unsigned_long_) {
return unsigned_long_val_;
} else if (is_string_) {
- return string_val_.empty() ? 0 : std::stoul(string_val_);
+ return string_val_.empty() ? 0 : std::stoull(string_val_);
} else if (is_signed_long_) {
return signed_long_val_;
} else if (is_long_double_) {
@@ -196,7 +196,7 @@ class Value {
} else if (is_unsigned_long_) {
return unsigned_long_val_;
} else if (is_string_) {
- return string_val_.empty() ? 0 : std::stol(string_val_);
+ return string_val_.empty() ? 0 : std::stoll(string_val_);
} else if (is_long_double_) {
return static_cast<int64_t >(long_double_val_);
} else {
diff --git a/extensions/expression-language/impl/expression/Expression.h b/extensions/expression-language/impl/expression/Expression.h
index 8e896087a..dd94a59fc 100644
--- a/extensions/expression-language/impl/expression/Expression.h
+++ b/extensions/expression-language/impl/expression/Expression.h
@@ -17,15 +17,6 @@
#pragma once
-#define EXPRESSION_LANGUAGE_USE_REGEX
-
-// Disable regex in EL for incompatible compilers
-#if !defined(WIN32) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 9))
-#undef EXPRESSION_LANGUAGE_USE_REGEX
-#endif
-
-#define EXPRESSION_LANGUAGE_USE_DATE
-
#include <string>
#include <memory>
#include <functional>
@@ -189,6 +180,10 @@ Expression make_dynamic_function(const std::string &function_name, const std::ve
*/
Expression make_function_composition(const Expression &arg, const std::vector<std::pair<std::string, std::vector<Expression>>> &chain);
+#ifdef WIN32
+void dateSetInstall(const std::string& install);
+#endif
+
} /* namespace expression */
} /* namespace minifi */
} /* namespace nifi */
diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt
index d2d2d8d2b..2214ef57b 100644
--- a/extensions/expression-language/tests/CMakeLists.txt
+++ b/extensions/expression-language/tests/CMakeLists.txt
@@ -21,33 +21,31 @@
file(GLOB EXPRESSION_LANGUAGE_TESTS "*.cpp")
SET(EXTENSIONS_TEST_COUNT 0)
-if(NOT WIN32)
- FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS})
- get_filename_component(testfilename "${testfile}" NAME_WE)
- add_executable(${testfilename} "${testfile}")
- target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/catch")
- target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test")
- target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include")
- target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors")
- target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors")
- target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language")
- createTests(${testfilename})
- target_link_libraries(${testfilename} ${CATCH_MAIN_LIB})
- if(NOT DISABLE_CURL)
- target_link_libraries(${testfilename} CURL::libcurl)
- endif()
- target_link_libraries(${testfilename} minifi-expression-language-extensions)
- target_link_libraries(${testfilename} minifi-standard-processors)
+FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS})
+ get_filename_component(testfilename "${testfile}" NAME_WE)
+ add_executable(${testfilename} "${testfile}")
+ target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/catch")
+ target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test")
+ target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include")
+ target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors")
+ target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors")
+ target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language")
+ createTests(${testfilename})
+ target_link_libraries(${testfilename} ${CATCH_MAIN_LIB})
+ if(NOT DISABLE_CURL)
+ target_link_libraries(${testfilename} CURL::libcurl)
+ endif()
+ target_link_libraries(${testfilename} minifi-expression-language-extensions)
+ target_link_libraries(${testfilename} minifi-standard-processors)
+ target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata")
- MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1")
- add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY ${TEST_DIR})
- ENDFOREACH()
-endif()
+ MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1")
+ add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY ${TEST_DIR})
+ENDFOREACH()
### integration tests
-if(NOT WIN32)
file(GLOB INT_EXPRESSION_LANGUAGE_TESTS "integration/*.cpp")
SET(INT_EXTENSIONS_TEST_COUNT 0)
@@ -74,4 +72,3 @@ ENDFOREACH()
add_test(NAME UpdateAttributeIntegrationTest COMMAND UpdateAttributeIntegrationTest "${TEST_RESOURCES}/TestUpdateAttribute.yml" "${TEST_RESOURCES}/")
message("-- Finished building ${EXTENSIONS_TEST_COUNT} expression language related test file(s)...")
-endif()
diff --git a/extensions/expression-language/tests/ExpressionLanguageTests.cpp b/extensions/expression-language/tests/ExpressionLanguageTests.cpp
index 8e657f0af..7b03fb5d1 100644
--- a/extensions/expression-language/tests/ExpressionLanguageTests.cpp
+++ b/extensions/expression-language/tests/ExpressionLanguageTests.cpp
@@ -40,6 +40,7 @@
#include "TestBase.h"
#include "Catch.h"
#include "unit/ProvenanceTestHelper.h"
+#include "date/tz.h"
namespace expression = org::apache::nifi::minifi::expression;
@@ -389,8 +390,6 @@ TEST_CASE("Substring After No Args", "[expressionLanguageSubstringAfterNoArgs]")
REQUIRE_THROWS_WITH(expression::compile("${attr:substringAfter()}"), "Expression language function substringAfter called with 1 argument(s), but 2 are required");
}
-#ifdef EXPRESSION_LANGUAGE_USE_REGEX
-
TEST_CASE("Replace", "[expressionLanguageReplace]") {
auto expr = expression::compile("${attr:replace('.', '_')}");
@@ -599,8 +598,6 @@ TEST_CASE("LastIndexOf4", "[expressionLanguageLastIndexOf4]") {
REQUIRE("11" == expr(expression::Parameters{ flow_file_a }).asString());
}
-#endif // EXPRESSION_LANGUAGE_USE_REGEX
-
TEST_CASE("Plus Integer", "[expressionLanguagePlusInteger]") {
auto expr = expression::compile("${attr:plus(13)}");
@@ -630,7 +627,7 @@ TEST_CASE("Plus Exponent 2", "[expressionLanguagePlusExponent2]") {
auto flow_file_a = std::make_shared<core::FlowFile>();
flow_file_a->addAttribute("attr", "11.345678901234");
- REQUIRE("10000011.345678901234351" == expr(expression::Parameters{ flow_file_a }).asString());
+ REQUIRE(10000011.345678901234 == Approx(expr(expression::Parameters{ flow_file_a }).asLongDouble()));
}
TEST_CASE("Minus Integer", "[expressionLanguageMinusInteger]") {
@@ -662,7 +659,7 @@ TEST_CASE("Multiply Decimal", "[expressionLanguageMultiplyDecimal]") {
auto flow_file_a = std::make_shared<core::FlowFile>();
flow_file_a->addAttribute("attr", "11.1");
- REQUIRE("-148.136937" == expr(expression::Parameters{ flow_file_a }).asString());
+ REQUIRE(-148.136937 == Approx(expr(expression::Parameters{ flow_file_a }).asLongDouble()));
}
TEST_CASE("Divide Integer", "[expressionLanguageDivideInteger]") {
@@ -1176,7 +1173,6 @@ TEST_CASE("Encode Decode CSV", "[expressionEncodeDecodeCSV]") {
REQUIRE("Zero > One < \"two!\" & 'true'" == expr(expression::Parameters{ flow_file_a }).asString());
}
-#ifndef WIN32
#ifndef DISABLE_CURL
TEST_CASE("Encode URL", "[expressionEncodeURL]") {
auto expr = expression::compile("${message:urlEncode()}");
@@ -1226,11 +1222,11 @@ TEST_CASE("Encode Decode URL", "[expressionEncodeDecodeURLExcept]") {
REQUIRE_THROWS(expr(expression::Parameters{flow_file_a}).asString());
}
#endif
-#endif
-
-#ifdef EXPRESSION_LANGUAGE_USE_DATE
TEST_CASE("Parse Date", "[expressionParseDate]") {
+#ifdef WIN32
+ expression::dateSetInstall(TZ_DATA_DIR);
+#endif
auto expr = expression::compile("${message:toDate('%Y/%m/%d', 'America/Los_Angeles')}");
auto flow_file_a = std::make_shared<core::FlowFile>();
@@ -1239,6 +1235,9 @@ TEST_CASE("Parse Date", "[expressionParseDate]") {
}
TEST_CASE("Reformat Date", "[expressionReformatDate]") {
+#ifdef WIN32
+ expression::dateSetInstall(TZ_DATA_DIR);
+#endif
auto expr = expression::compile("${message:toDate('%Y/%m/%d', 'GMT'):format('%m-%d-%Y', 'America/New_York')}");
auto flow_file_a = std::make_shared<core::FlowFile>();
@@ -1246,18 +1245,17 @@ TEST_CASE("Reformat Date", "[expressionReformatDate]") {
REQUIRE("03-13-2014" == expr(expression::Parameters{ flow_file_a }).asString());
}
-#endif // EXPRESSION_LANGUAGE_USE_DATE
-
TEST_CASE("Now Date", "[expressionNowDate]") {
+#ifdef WIN32
+ expression::dateSetInstall(TZ_DATA_DIR);
+#endif
auto expr = expression::compile("${now():format('%Y')}");
auto flow_file_a = std::make_shared<core::FlowFile>();
flow_file_a->addAttribute("message", "2014/03/14");
- time_t t = time(nullptr);
- struct tm lt;
- localtime_r(&t, <);
+ date::year_month_day date{std::chrono::floor<std::chrono::days>(std::chrono::system_clock::now())};
- REQUIRE(gsl::narrow<uint64_t>(lt.tm_year + 1900) == expr(expression::Parameters{ flow_file_a }).asUnsignedLong());
+ REQUIRE(date.year().operator int() == expr(expression::Parameters{ flow_file_a }).asSignedLong());
}
TEST_CASE("Format Date", "[expressionFormatDate]") {
@@ -1383,8 +1381,6 @@ TEST_CASE("Any Contains 2", "[expressionAnyContains2]") {
REQUIRE(!expr(expression::Parameters{ flow_file_a }).asBoolean());
}
-#ifdef EXPRESSION_LANGUAGE_USE_REGEX
-
TEST_CASE("All Matching Contains", "[expressionAllMatchingContains]") {
auto expr = expression::compile("${allMatchingAttributes('xyz_.*'):contains('hello')}");
@@ -1457,8 +1453,6 @@ TEST_CASE("Any Matching Contains 4", "[expressionAnyMatchingContains4]") {
REQUIRE(!expr(expression::Parameters{ flow_file_a }).asBoolean());
}
-#endif // EXPRESSION_LANGUAGE_USE_REGEX
-
TEST_CASE("All Delineated Contains", "[expressionAllDelineatedContains]") {
auto expr = expression::compile("${allDelineatedValues(${word_list}, \",\"):contains('hello')}");
diff --git a/extensions/expression-language/tests/ProcessContextExprTests.cpp b/extensions/expression-language/tests/ProcessContextExprTests.cpp
index 46da66fae..1cef4eabc 100644
--- a/extensions/expression-language/tests/ProcessContextExprTests.cpp
+++ b/extensions/expression-language/tests/ProcessContextExprTests.cpp
@@ -31,8 +31,8 @@ namespace org::apache::nifi::minifi {
class DummyProcessor : public core::Processor {
public:
using core::Processor::Processor;
- EXTENSIONAPI static core::Property SimpleProperty;
- EXTENSIONAPI static core::Property ExpressionLanguageProperty;
+ static core::Property SimpleProperty;
+ static core::Property ExpressionLanguageProperty;
void initialize() override { setSupportedProperties({SimpleProperty, ExpressionLanguageProperty}); }
bool supportsDynamicProperties() override { return true; }
};