You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/12/05 17:33:48 UTC
[arrow] branch master updated: ARROW-3441: [Gandiva] Use common
unit test creation facilities,
do not produce multiple executables for the same unit tests
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8db3748 ARROW-3441: [Gandiva] Use common unit test creation facilities, do not produce multiple executables for the same unit tests
8db3748 is described below
commit 8db3748f29ccacfee2f0b5a36d102e68d2e008f4
Author: Wes McKinney <we...@apache.org>
AuthorDate: Wed Dec 5 11:33:22 2018 -0600
ARROW-3441: [Gandiva] Use common unit test creation facilities, do not produce multiple executables for the same unit tests
This also adds a "gandiva" CMake target so you can run `ninja gandiva`. Additionally, all unit tests now have a `gandiva-` prefix.
Author: Wes McKinney <we...@apache.org>
Closes #3091 from wesm/ARROW-3441 and squashes the following commits:
d6f4703de <Wes McKinney> Add option to ADD_ARROW_TESTS to override global NO_TESTS option
4f006feb3 <Wes McKinney> Add prefix for precompiled tests, add to gandiva target
5bdfd3090 <Wes McKinney> Add ADD_GANDIVA_TEST function, remove GandivaBuildUtils, cleaning. Add gandiva custom target
---
cpp/cmake_modules/BuildUtils.cmake | 11 ++--
cpp/cmake_modules/GandivaBuildUtils.cmake | 91 ------------------------------
cpp/src/arrow/ipc/CMakeLists.txt | 3 +-
cpp/src/gandiva/CMakeLists.txt | 83 +++++++++++++++++++--------
cpp/src/gandiva/jni/CMakeLists.txt | 3 +
cpp/src/gandiva/precompiled/CMakeLists.txt | 17 ++++++
cpp/src/gandiva/tests/CMakeLists.txt | 37 ++++++------
7 files changed, 103 insertions(+), 142 deletions(-)
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index fb646dd..916b9eb 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -368,6 +368,7 @@ endfunction()
#
# Arguments after the test name will be passed to set_tests_properties().
#
+# \arg ENABLED if passed, add this unit test even if ARROW_BUILD_TESTS is off
# \arg PREFIX a string to append to the name of the test executable. For
# example, if you have src/arrow/foo/bar-test.cc, then PREFIX "foo" will create
# test executable foo-bar-test
@@ -377,7 +378,7 @@ endfunction()
# groups using the syntax unittest;GROUP2;GROUP3. Custom targets for the group
# names must exist
function(ADD_ARROW_TEST REL_TEST_NAME)
- set(options NO_VALGRIND)
+ set(options NO_VALGRIND ENABLED)
set(one_value_args)
set(multi_value_args SOURCES STATIC_LINK_LIBS EXTRA_LINK_LIBS EXTRA_INCLUDES
EXTRA_DEPENDENCIES LABELS PREFIX)
@@ -398,7 +399,7 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
endif()
endif()
- if (NO_TESTS)
+ if (NO_TESTS AND NOT ARG_ENABLED)
return()
endif()
get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
@@ -424,13 +425,13 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
if (ARG_STATIC_LINK_LIBS)
# Customize link libraries
- target_link_libraries(${TEST_NAME} ${ARG_STATIC_LINK_LIBS})
+ target_link_libraries(${TEST_NAME} PRIVATE ${ARG_STATIC_LINK_LIBS})
else()
- target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+ target_link_libraries(${TEST_NAME} PRIVATE ${ARROW_TEST_LINK_LIBS})
endif()
if (ARG_EXTRA_LINK_LIBS)
- target_link_libraries(${TEST_NAME} ${ARG_EXTRA_LINK_LIBS})
+ target_link_libraries(${TEST_NAME} PRIVATE ${ARG_EXTRA_LINK_LIBS})
endif()
if (ARG_EXTRA_INCLUDES)
diff --git a/cpp/cmake_modules/GandivaBuildUtils.cmake b/cpp/cmake_modules/GandivaBuildUtils.cmake
deleted file mode 100644
index 521d697..0000000
--- a/cpp/cmake_modules/GandivaBuildUtils.cmake
+++ /dev/null
@@ -1,91 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(GANDIVA_TEST_LINK_LIBS
- gtest_static
- gtest_main_static
- ${RE2_LIBRARY})
-
-if (PTHREAD_LIBRARY)
- set(GANDIVA_TEST_LINK_LIBS
- ${GANDIVA_TEST_LINK_LIBS}
- ${PTHREAD_LIBRARY})
-endif()
-
-# Add a unittest executable, with its dependencies.
-function(add_gandiva_unit_test REL_TEST_NAME)
- get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
- add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN})
- if(${REL_TEST_NAME} MATCHES "llvm" OR
- ${REL_TEST_NAME} MATCHES "expression_registry")
- # If the unit test has llvm in its name, include llvm.
- add_dependencies(${TEST_NAME} LLVM::LLVM_INTERFACE)
- target_link_libraries(${TEST_NAME} PRIVATE LLVM::LLVM_INTERFACE)
- endif()
-
- # Require toolchain to be built
- add_dependencies(${TEST_NAME} arrow_dependencies)
-
- target_include_directories(${TEST_NAME} PRIVATE
- ${CMAKE_SOURCE_DIR}/include
- ${CMAKE_SOURCE_DIR}/src
- )
- target_link_libraries(${TEST_NAME}
- PRIVATE arrow_shared ${GANDIVA_TEST_LINK_LIBS}
- )
- add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
- set_property(TEST ${TEST_NAME} PROPERTY LABELS gandiva,unittest ${TEST_NAME})
-endfunction(add_gandiva_unit_test REL_TEST_NAME)
-
-# Add a unittest executable for a precompiled file (used to generate IR)
-function(add_precompiled_unit_test REL_TEST_NAME)
- get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
- add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN})
- # Require toolchain to be built
- add_dependencies(${TEST_NAME} arrow_dependencies)
- target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/src)
- target_link_libraries(${TEST_NAME}
- PRIVATE arrow_shared ${GANDIVA_TEST_LINK_LIBS}
- )
- target_compile_definitions(${TEST_NAME} PRIVATE GANDIVA_UNIT_TEST=1)
- add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
- set_property(TEST ${TEST_NAME} PROPERTY LABELS gandiva,unittest ${TEST_NAME})
-endfunction(add_precompiled_unit_test REL_TEST_NAME)
-
-# Add an integ executable, with its dependencies.
-function(add_gandiva_integ_test REL_TEST_NAME GANDIVA_LIB)
- get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
- add_executable(${TEST_NAME}_${GANDIVA_LIB} ${REL_TEST_NAME} ${ARGN})
- target_include_directories(${TEST_NAME}_${GANDIVA_LIB} PRIVATE ${CMAKE_SOURCE_DIR})
- target_link_libraries(${TEST_NAME}_${GANDIVA_LIB} PRIVATE
- ${GANDIVA_LIB}
- ${GANDIVA_TEST_LINK_LIBS}
- )
-
- add_test(NAME ${TEST_NAME}_${GANDIVA_LIB} COMMAND ${TEST_NAME}_${GANDIVA_LIB})
- set_property(TEST ${TEST_NAME}_${GANDIVA_LIB} PROPERTY LABELS gandiva,integ ${TEST_NAME}_${GANDIVA_LIB})
-endfunction(add_gandiva_integ_test REL_TEST_NAME)
-
-function(prevent_in_source_builds)
- file(TO_CMAKE_PATH "${PROJECT_BINARY_DIR}/CMakeLists.txt" LOC_PATH)
- if(EXISTS "${LOC_PATH}")
- message(FATAL_ERROR "Gandiva does not support in-source builds")
- endif()
-endfunction(prevent_in_source_builds)
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 13ed9b9..9c384c3 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -33,8 +33,7 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
set_target_properties(json-integration-test
PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
else()
- target_link_libraries(json-integration-test
- pthread)
+ target_link_libraries(json-integration-test PRIVATE pthread)
endif()
endif()
endif()
diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt
index b71313e..bd497dc 100644
--- a/cpp/src/gandiva/CMakeLists.txt
+++ b/cpp/src/gandiva/CMakeLists.txt
@@ -20,10 +20,11 @@ cmake_minimum_required(VERSION 3.11)
project(gandiva)
-include(GandivaBuildUtils)
-
find_package(LLVM)
+# For "make gandiva" to build everything Gandiva-related
+add_custom_target(gandiva)
+
# Set the path where the byte-code files will be installed.
set(GANDIVA_BC_INSTALL_DIR
${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/gandiva)
@@ -92,6 +93,8 @@ ADD_ARROW_LIB(gandiva
SHARED_PRIVATE_LINK_LIBS ${GANDIVA_SHARED_PRIVATE_LINK_LIBS}
STATIC_LINK_LIBS ${GANDIVA_STATIC_LINK_LIBS})
+add_dependencies(gandiva ${GANDIVA_LIBRARIES})
+
# install for gandiva
include(GNUInstallDirs)
@@ -125,28 +128,62 @@ install(
FILES "${CMAKE_CURRENT_BINARY_DIR}/gandiva.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
+set(GANDIVA_STATIC_TEST_LINK_LIBS
+ gandiva_static
+ ${RE2_LIBRARY}
+ ${ARROW_TEST_LINK_LIBS})
+set(GANDIVA_SHARED_TEST_LINK_LIBS
+ gandiva_shared
+ ${RE2_LIBRARY}
+ ${ARROW_TEST_LINK_LIBS})
+
+function(ADD_GANDIVA_TEST REL_TEST_NAME)
+ set(options USE_STATIC_LINKING)
+ set(one_value_args)
+ set(multi_value_args)
+ cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+
+ set(TEST_ARGUMENTS
+ ENABLED
+ PREFIX "gandiva"
+ LABELS "unittest;gandiva"
+ ${ARG_UNPARSED_ARGUMENTS})
+
+ # and uses less disk space, but in some cases we need to force static
+ # linking (see rationale below).
+ if (ARG_USE_STATIC_LINKING)
+ ADD_ARROW_TEST(${REL_TEST_NAME}
+ ${TEST_ARGUMENTS}
+ STATIC_LINK_LIBS ${GANDIVA_STATIC_TEST_LINK_LIBS})
+ else()
+ ADD_ARROW_TEST(${REL_TEST_NAME}
+ ${TEST_ARGUMENTS}
+ STATIC_LINK_LIBS ${GANDIVA_SHARED_TEST_LINK_LIBS})
+ endif()
+
+ if(${REL_TEST_NAME} MATCHES "llvm" OR
+ ${REL_TEST_NAME} MATCHES "expression_registry")
+ # If the unit test has llvm in its name, include llvm.
+ add_dependencies(gandiva-${REL_TEST_NAME} LLVM::LLVM_INTERFACE)
+ target_link_libraries(gandiva-${REL_TEST_NAME} PRIVATE LLVM::LLVM_INTERFACE)
+ endif()
+endfunction()
+
if (ARROW_GANDIVA_BUILD_TESTS)
- #args: label test-file src-files
- add_gandiva_unit_test(bitmap_accumulator_test.cc bitmap_accumulator.cc)
- add_gandiva_unit_test(engine_llvm_test.cc engine.cc llvm_types.cc configuration.cc
- gdv_function_stubs.cc context_helper.cc to_date_holder.cc date_utils.cc
- exported_funcs_registry.cc ${BC_FILE_PATH_CC})
- add_gandiva_unit_test(function_signature_test.cc function_signature.cc)
- add_gandiva_unit_test(function_registry_test.cc function_registry.cc function_signature.cc)
- add_gandiva_unit_test(llvm_types_test.cc llvm_types.cc)
- add_gandiva_unit_test(llvm_generator_test.cc llvm_generator.cc regex_util.cc engine.cc
- llvm_types.cc expr_decomposer.cc function_registry.cc annotator.cc
- bitmap_accumulator.cc configuration.cc function_signature.cc like_holder.cc
- to_date_holder.cc date_utils.cc regex_util.cc gdv_function_stubs.cc context_helper.cc
- exported_funcs_registry.cc ${BC_FILE_PATH_CC})
- add_gandiva_unit_test(annotator_test.cc annotator.cc function_signature.cc)
- add_gandiva_unit_test(tree_expr_test.cc tree_expr_builder.cc expr_decomposer.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc to_date_holder.cc date_utils.cc)
- add_gandiva_unit_test(expr_decomposer_test.cc expr_decomposer.cc tree_expr_builder.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc to_date_holder.cc date_utils.cc)
- add_gandiva_unit_test(expression_registry_test.cc llvm_types.cc expression_registry.cc function_signature.cc function_registry.cc)
- add_gandiva_unit_test(selection_vector_test.cc selection_vector.cc)
- add_gandiva_unit_test(lru_cache_test.cc)
- add_gandiva_unit_test(to_date_holder_test.cc to_date_holder.cc date_utils.cc)
- add_gandiva_unit_test(simple_arena_test.cc)
+ ADD_GANDIVA_TEST(bitmap_accumulator_test)
+ ADD_GANDIVA_TEST(engine_llvm_test)
+ ADD_GANDIVA_TEST(function_signature_test)
+ ADD_GANDIVA_TEST(function_registry_test)
+ ADD_GANDIVA_TEST(llvm_types_test)
+ ADD_GANDIVA_TEST(llvm_generator_test)
+ ADD_GANDIVA_TEST(annotator_test)
+ ADD_GANDIVA_TEST(tree_expr_test)
+ ADD_GANDIVA_TEST(expr_decomposer_test)
+ ADD_GANDIVA_TEST(expression_registry_test)
+ ADD_GANDIVA_TEST(selection_vector_test)
+ ADD_GANDIVA_TEST(lru_cache_test)
+ ADD_GANDIVA_TEST(to_date_holder_test)
+ ADD_GANDIVA_TEST(simple_arena_test)
endif()
if (ARROW_GANDIVA_JAVA)
diff --git a/cpp/src/gandiva/jni/CMakeLists.txt b/cpp/src/gandiva/jni/CMakeLists.txt
index 8684fe8..9f7bc52 100644
--- a/cpp/src/gandiva/jni/CMakeLists.txt
+++ b/cpp/src/gandiva/jni/CMakeLists.txt
@@ -61,6 +61,7 @@ set(GANDIVA_JNI_SOURCES config_builder.cc
# cpp/src
ADD_ARROW_LIB(gandiva_jni
SOURCES ${GANDIVA_JNI_SOURCES}
+ OUTPUTS GANDIVA_JNI_LIBRARIES
SHARED_PRIVATE_LINK_LIBS ${GANDIVA_LINK_LIBS}
STATIC_LINK_LIBS ${GANDIVA_LINK_LIBS}
DEPENDENCIES gandiva_java gandiva_jni_proto
@@ -69,6 +70,8 @@ ADD_ARROW_LIB(gandiva_jni
${JNI_HEADERS_DIR}
PRIVATE_INCLUDES ${JNI_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})
+add_dependencies(gandiva ${GANDIVA_JNI_LIBRARIES})
+
# filter out everything that is not needed for the jni bridge
# statically linked stdc++ has conflicts with stdc++ loaded by other libraries.
if (NOT APPLE)
diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt
index 46e80d5..886fdce 100644
--- a/cpp/src/gandiva/precompiled/CMakeLists.txt
+++ b/cpp/src/gandiva/precompiled/CMakeLists.txt
@@ -51,6 +51,23 @@ add_custom_command(
add_custom_target(precompiled ALL DEPENDS ${GANDIVA_BC_OUTPUT_PATH})
+# Add a unittest executable for a precompiled file (used to generate IR)
+function(add_precompiled_unit_test REL_TEST_NAME)
+ get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+ set(TEST_NAME "gandiva-precompiled-${TEST_NAME}")
+
+ add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN})
+ add_dependencies(gandiva ${TEST_NAME})
+ target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/src)
+ target_link_libraries(${TEST_NAME}
+ PRIVATE ${ARROW_TEST_LINK_LIBS} ${RE2_LIBRARY}
+ )
+ target_compile_definitions(${TEST_NAME} PRIVATE GANDIVA_UNIT_TEST=1)
+ add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
+ set_property(TEST ${TEST_NAME} PROPERTY LABELS gandiva;unittest ${TEST_NAME})
+endfunction(add_precompiled_unit_test REL_TEST_NAME)
+
# testing
if (ARROW_GANDIVA_BUILD_TESTS)
add_precompiled_unit_test(bitmap_test.cc bitmap.cc)
diff --git a/cpp/src/gandiva/tests/CMakeLists.txt b/cpp/src/gandiva/tests/CMakeLists.txt
index ae60063..1fd30aa 100644
--- a/cpp/src/gandiva/tests/CMakeLists.txt
+++ b/cpp/src/gandiva/tests/CMakeLists.txt
@@ -15,28 +15,23 @@
# specific language governing permissions and limitations
# under the License.
-project(gandiva)
+ADD_GANDIVA_TEST(filter_test)
+ADD_GANDIVA_TEST(projector_test)
+ADD_GANDIVA_TEST(projector_build_validation_test)
+ADD_GANDIVA_TEST(if_expr_test)
+ADD_GANDIVA_TEST(literal_test)
+ADD_GANDIVA_TEST(boolean_expr_test)
+ADD_GANDIVA_TEST(binary_test)
+ADD_GANDIVA_TEST(date_time_test)
+ADD_GANDIVA_TEST(to_string_test)
+ADD_GANDIVA_TEST(hash_test)
+ADD_GANDIVA_TEST(in_expr_test)
+ADD_GANDIVA_TEST(null_validity_test)
-foreach(lib_type "shared" "static")
- add_gandiva_integ_test(filter_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(projector_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(if_expr_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(literal_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(projector_build_validation_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(boolean_expr_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(utf8_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(binary_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(date_time_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(to_string_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(hash_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(in_expr_test.cc gandiva_${lib_type})
- add_gandiva_integ_test(null_validity_test.cc gandiva_${lib_type})
-endforeach(lib_type)
-
-set(GANDIVA_BENCHMARK_LINK_LIBRARIES
- gandiva_static
-)
+ADD_GANDIVA_TEST(projector_test_static
+ SOURCES projector_test.cc
+ USE_STATIC_LINKING)
ADD_ARROW_BENCHMARK(micro_benchmarks
PREFIX "gandiva"
- EXTRA_LINK_LIBS ${GANDIVA_BENCHMARK_LINK_LIBRARIES})
+ EXTRA_LINK_LIBS gandiva_static)