You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2015/07/01 12:46:12 UTC
[41/50] hadoop git commit: HDFS-8635. Migrate HDFS native build to
new CMake framework (Alan Burlison via Colin P. McCabe)
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29df73dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29df73dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29df73dc
Branch: refs/heads/YARN-2139
Commit: 29df73dc0572cf8e450568f1d0301e8d55d64b48
Parents: 9ee7b6e
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 15:54:38 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jun 30 15:54:38 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop-hdfs/src/CMakeLists.txt | 129 ++++++-------------
.../src/contrib/libwebhdfs/CMakeLists.txt | 8 +-
.../libwebhdfs/resources/FindJansson.cmake | 9 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 91 +++++++------
5 files changed, 100 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b065f98..65f286d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8493. Consolidate truncate() related implementation in a single class.
(Rakesh R via wheat9)
+ HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
+ via Colin P. McCabe)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 563727b..b4a3b40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -18,106 +18,59 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
- add_library(${LIBNAME} SHARED ${ARGN})
- add_library(${LIBNAME}_static STATIC ${ARGN})
- # Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so
- # (dynamic linking). On Windows, we cannot use the same base name for both
- # static and dynamic, because Windows does not use distinct file extensions
- # for a statically linked library vs. a DLL import library. Both use the
- # .lib extension. On Windows, we'll build the static library as
- # hdfs_static.lib.
- if (NOT WIN32)
- set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
- endif (NOT WIN32)
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
- target_link_libraries(${LIBNAME} ${ARGN})
- target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
- output_directory(${TGT} "${DIR}")
- output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
+#
+# Main configuration
+#
-# Flatten a list into a string.
-function(FLATTEN_LIST INPUT SEPARATOR OUTPUT)
- string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
- set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
-endfunction()
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+ message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
# Check to see if our compiler and linker support the __thread attribute.
-# On Linux and some other operating systems, this is a more efficient
-# alternative to POSIX thread local storage.
-INCLUDE(CheckCSourceCompiles)
-CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+# On Linux and some other operating systems, this is a more efficient
+# alternative to POSIX thread local storage.
+include(CheckCSourceCompiles)
+check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
# Check to see if we have Intel SSE intrinsics.
-CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
+check_c_source_compileS("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
# Check if we need to link dl library to get dlopen.
# dlopen on Linux is in separate library but on FreeBSD its in libc
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
-find_package(JNI REQUIRED)
-if (NOT GENERATED_JAVAH)
- # Must identify where the generated headers have been placed
- MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-
-if (WIN32)
+if(WIN32)
+ # Set the optimizer level.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2")
-
# Set warning level 4.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
-
# Skip "unreferenced formal parameter".
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100")
-
# Skip "conditional expression is constant".
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127")
-
# Skip deprecated POSIX function warnings.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE")
-
# Skip CRT non-secure function warnings. If we can convert usage of
# strerror, getenv and ctime to their secure CRT equivalents, then we can
# re-enable the CRT non-secure function warnings.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS")
-
# Omit unneeded headers.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
-
set(OS_DIR main/native/libhdfs/os/windows)
set(OUT_DIR target/bin)
-else (WIN32)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
+else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(OS_DIR main/native/libhdfs/os/posix)
- set(OS_LINK_LIBRARIES pthread)
set(OUT_DIR target/usr/local/lib)
-endif (WIN32)
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
add_definitions(-DLIBHDFS_DLL_EXPORT)
@@ -132,9 +85,9 @@ include_directories(
)
set(_FUSE_DFS_VERSION 0.1.0)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
-add_dual_library(hdfs
+hadoop_add_dual_library(hdfs
main/native/libhdfs/exception.c
main/native/libhdfs/jni_helper.c
main/native/libhdfs/hdfs.c
@@ -142,17 +95,17 @@ add_dual_library(hdfs
${OS_DIR}/mutexes.c
${OS_DIR}/thread_local_storage.c
)
-if (NEED_LINK_DL)
+if(NEED_LINK_DL)
set(LIB_DL dl)
-endif(NEED_LINK_DL)
+endif()
-target_link_dual_libraries(hdfs
+hadoop_target_link_dual_libraries(hdfs
${JAVA_JVM_LIBRARY}
${LIB_DL}
${OS_LINK_LIBRARIES}
)
-dual_output_directory(hdfs ${OUT_DIR})
+hadoop_dual_output_directory(hdfs ${OUT_DIR})
set(LIBHDFS_VERSION "0.0.0")
set_target_properties(hdfs PROPERTIES
SOVERSION ${LIBHDFS_VERSION})
@@ -232,25 +185,21 @@ target_link_libraries(test_htable
# Skip vecsum on Windows. This could be made to work in the future by
# introducing an abstraction layer over the sys/mman.h functions.
-if (NOT WIN32)
- add_executable(test_libhdfs_vecsum
- main/native/libhdfs/test/vecsum.c
- )
- if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(NOT WIN32)
+ add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c)
+ if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(test_libhdfs_vecsum
hdfs
- pthread
- )
- else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ pthread)
+ else()
target_link_libraries(test_libhdfs_vecsum
hdfs
pthread
- rt
- )
- endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-endif(NOT WIN32)
+ rt)
+ endif()
+endif()
-IF(REQUIRE_LIBWEBHDFS)
+if(REQUIRE_LIBWEBHDFS)
add_subdirectory(contrib/libwebhdfs)
-ENDIF(REQUIRE_LIBWEBHDFS)
+endif()
add_subdirectory(main/native/fuse-dfs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
index 44f0c42..dc74feb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
@@ -19,12 +19,12 @@
find_package(CURL REQUIRED)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
+ "${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
find_package(Jansson REQUIRED)
include_directories(${JANSSON_INCLUDE_DIR})
-add_dual_library(webhdfs
+hadoop_add_dual_library(webhdfs
src/hdfs_web.c
src/hdfs_http_client.c
src/hdfs_http_query.c
@@ -35,13 +35,13 @@ add_dual_library(webhdfs
../../${OS_DIR}/thread_local_storage.c
../../main/native/libhdfs/common/htable.c
)
-target_link_dual_libraries(webhdfs
+hadoop_target_link_dual_libraries(webhdfs
${JAVA_JVM_LIBRARY}
${CURL_LIBRARY}
${JANSSON_LIBRARY}
pthread
)
-dual_output_directory(webhdfs target)
+hadoop_dual_output_directory(webhdfs target)
set(LIBWEBHDFS_VERSION "0.0.0")
set_target_properties(webhdfs PROPERTIES
SOVERSION ${LIBWEBHDFS_VERSION})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
index cb8c8db..b8c67ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
@@ -5,9 +5,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,8 @@
# JANSSON_DEFINITIONS - Compiler switches required for using Jansson
find_path(JANSSON_INCLUDE_DIR jansson.h
- /usr/incluce
+ /usr/include
+ /usr/include/jansson
/usr/local/include )
find_library(JANSSON_LIBRARY NAMES jansson
@@ -39,4 +40,4 @@ include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Jansson DEFAULT_MSG
JANSSON_LIBRARY JANSSON_INCLUDE_DIR)
-mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
\ No newline at end of file
+mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
index 8a8ea5b..7a5e5a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
@@ -18,57 +18,63 @@
set(CMAKE_SKIP_RPATH TRUE)
+# Flatten a list into a string.
+function(flatten_list INPUT SEPARATOR OUTPUT)
+ string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
+ set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
+endfunction()
+
# Find Linux FUSE
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
find_package(PkgConfig REQUIRED)
pkg_check_modules(FUSE fuse)
- IF(FUSE_FOUND)
- FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
- FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
+ if(FUSE_FOUND)
+ flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
+ flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}")
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}")
- MESSAGE(STATUS "Building Linux FUSE client.")
+ message(STATUS "Building Linux FUSE client.")
include_directories(${FUSE_INCLUDE_DIRS})
- ELSE(FUSE_FOUND)
- MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
- ENDIF(FUSE_FOUND)
-ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.")
-ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+ else()
+ message(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
+ endif(FUSE_FOUND)
+else()
+ message(STATUS "Non-Linux system detected. Will not build FUSE client.")
+endif()
-IF(FUSE_FOUND)
+if(FUSE_FOUND)
add_library(posix_util
../util/posix_util.c
)
add_executable(fuse_dfs
fuse_dfs.c
- fuse_options.c
- fuse_connect.c
- fuse_impls_access.c
- fuse_impls_chmod.c
- fuse_impls_chown.c
- fuse_impls_create.c
- fuse_impls_flush.c
- fuse_impls_getattr.c
- fuse_impls_mkdir.c
- fuse_impls_mknod.c
- fuse_impls_open.c
- fuse_impls_read.c
- fuse_impls_readdir.c
- fuse_impls_release.c
- fuse_impls_rename.c
- fuse_impls_rmdir.c
- fuse_impls_statfs.c
- fuse_impls_symlink.c
- fuse_impls_truncate.c
- fuse_impls_unlink.c
- fuse_impls_utimens.c
+ fuse_options.c
+ fuse_connect.c
+ fuse_impls_access.c
+ fuse_impls_chmod.c
+ fuse_impls_chown.c
+ fuse_impls_create.c
+ fuse_impls_flush.c
+ fuse_impls_getattr.c
+ fuse_impls_mkdir.c
+ fuse_impls_mknod.c
+ fuse_impls_open.c
+ fuse_impls_read.c
+ fuse_impls_readdir.c
+ fuse_impls_release.c
+ fuse_impls_rename.c
+ fuse_impls_rmdir.c
+ fuse_impls_statfs.c
+ fuse_impls_symlink.c
+ fuse_impls_truncate.c
+ fuse_impls_unlink.c
+ fuse_impls_utimens.c
fuse_impls_write.c
- fuse_init.c
- fuse_stat_struct.c
- fuse_trash.c
- fuse_users.c
+ fuse_init.c
+ fuse_stat_struct.c
+ fuse_trash.c
+ fuse_users.c
)
target_link_libraries(fuse_dfs
${FUSE_LIBRARIES}
@@ -88,8 +94,9 @@ IF(FUSE_FOUND)
posix_util
pthread
)
-ELSE(FUSE_FOUND)
- IF(REQUIRE_FUSE)
- MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.")
- ENDIF(REQUIRE_FUSE)
-ENDIF(FUSE_FOUND)
+ set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning
+else()
+ if(REQUIRE_FUSE)
+ message(FATAL_ERROR "Required component fuse_dfs could not be built.")
+ endif()
+endif()