You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mj...@apache.org on 2017/07/28 22:36:55 UTC
incubator-impala git commit: IMPALA-4669: [KUTIL] Add kudu_util
library to the build.
Repository: incubator-impala
Updated Branches:
refs/heads/master fec05231b -> 1cbfa423b
IMPALA-4669: [KUTIL] Add kudu_util library to the build.
A few miscellaneous changes to allow kudu_util to compile with Impala.
Add kudu_version.cc to substitute for the version.cc file that is
automatically built during the full Kudu build.
Set LZ4_DISABLE_DEPRECATE_WARNINGS to allow Kudu's compressor utility to
use deprecated names for LZ4 methods.
Add NO_NVM_SUPPORT flag to Kudu build (plan to upstream this later) to
disable building with nvm support, removing a library dependency.
Also remove imported FindOpenSSL.cmake in favour of the standard one provided
by cmake itself.
Finally, a few changes to allow compilation on RHEL5:
* Only use sched_getcpu() if supported
* Only include magic.h if available
* Workaround for kernels that don't have SOCK_NONBLOCK
* Workaround for kernels that don't have O_CLOEXEC (ignore the flag)
* Provide non-working implementation of fallocate()
* Disable inclusion of linux/fiemap.h - although this exists on RHEL5,
it does not compile due to other #includes in env_posix.cc. We disable
the path this is used for, since Impala does not call that code.
* Use Kudu's implementation of pipe(2), preadv(2) and pwritev(2) where
it doesn't exist.
In most cases these changes simply force kutil to revert to a different
implementation that was already written for OSX support - this patch
generalises the logic to provide the implementation whenever the
required function doesn't exist.
This patch compiles on RHEL5.5 and 6.0, SLES11 and 12, Ubuntu 12.04 and
14.04 and Debian 7.0 and 8.0.
Change-Id: I8da798ee55506d6f969416b17c191eb03cb215f5
Reviewed-on: http://gerrit.cloudera.org:8080/5715
Tested-by: Impala Public Jenkins
Reviewed-by: Henry Robinson <he...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1cbfa423
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1cbfa423
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1cbfa423
Branch: refs/heads/master
Commit: 1cbfa423b4b884d160a607c7a1f359444cdd6aaf
Parents: fec0523
Author: Henry Robinson <he...@cloudera.com>
Authored: Tue Nov 8 12:06:48 2016 -0800
Committer: Henry Robinson <he...@cloudera.com>
Committed: Fri Jul 28 21:47:07 2017 +0000
----------------------------------------------------------------------
CMakeLists.txt | 8 ++-
be/CMakeLists.txt | 18 +++++-
be/src/common/CMakeLists.txt | 1 +
be/src/common/config.h.in | 7 ++-
be/src/common/kudu_version.cc | 34 ++++++++++++
be/src/common/logging.cc | 2 +
be/src/exec/kudu-util.h | 3 +
be/src/kudu/gutil | 1 +
be/src/kudu/util/CMakeLists.txt | 28 +++++++---
be/src/kudu/util/cache.cc | 2 +-
.../kudu/util/compression/compression_codec.cc | 6 ++
be/src/kudu/util/env.cc | 1 +
be/src/kudu/util/env_posix.cc | 23 +++++++-
be/src/kudu/util/flags.cc | 5 +-
be/src/kudu/util/kudu_export.h | 58 ++++++++++++++++++++
be/src/kudu/util/locks.h | 2 +-
be/src/kudu/util/logging.cc | 11 ++--
be/src/kudu/util/minidump.cc | 20 ++-----
be/src/kudu/util/net/socket.cc | 6 +-
be/src/kudu/util/subprocess.cc | 7 ++-
bin/run_clang_tidy.sh | 6 +-
21 files changed, 203 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 89cf708..e31e98f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,6 +17,11 @@
cmake_minimum_required(VERSION 3.2.3)
+# This is a Kudu-specific flag that disables Kudu targets that are test-only.
+set(NO_TESTS 1)
+# Kudu-specific, disable NVM support so that libvmem is not a dependency.
+set(NO_NVM_SUPPORT 1)
+
# Explicitly define project() to allow modifying the compiler before the project is
# initialized.
project(Impala)
@@ -273,7 +278,8 @@ IMPALA_ADD_THIRDPARTY_LIB(java_jvm "${JNI_INCLUDE_DIRS}" ${JAVA_JVM_LIBRARY} "")
# find breakpad headers and libs
find_package(Breakpad REQUIRED)
-IMPALA_ADD_THIRDPARTY_LIB(breakpad ${BREAKPAD_INCLUDE_DIR} ${BREAKPAD_STATIC_LIB} "")
+IMPALA_ADD_THIRDPARTY_LIB(breakpad_client ${BREAKPAD_INCLUDE_DIR} ${BREAKPAD_STATIC_LIB}
+ "")
# System dependencies
if (NOT APPLE)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 10e47f3..ae84528 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -172,6 +172,12 @@ set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${RULE_LAUNCH_PREFIX})
# Thrift requires these definitions for some types that we use
add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DHAVE_NETDB_H)
+# Kudu flags. 1. Enable non KUDU_* prefixes for status macros to allow Kudu code to
+# compile without changing the macro. 2. Enable full support for all backing types of
+# kudu::Slices. 3. Don't include stubs.h
+add_definitions(-DKUDU_HEADERS_USE_SHORT_STATUS_MACROS -DKUDU_HEADERS_USE_RICH_SLICE
+ -DKUDU_HEADERS_NO_STUBS)
+
# Set clang flags for cross-compiling to IR.
# IR_COMPILE is #defined for the cross compile to remove code that bloats the IR.
# We enable basic optimizations (-O1) to reduce the IR size and speed up runtime JIT.
@@ -290,6 +296,14 @@ EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
# Determine what functions are available on the current platform.
INCLUDE(CheckFunctionExists)
CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU)
+CHECK_FUNCTION_EXISTS(pipe2 HAVE_PIPE2)
+
+# linux/fs.h defines HAVE_FALLOCATE whether or not the function is available,
+# which is why we use IMPALA_HAVE_FALLOCATE here.
+CHECK_FUNCTION_EXISTS(fallocate IMPALA_HAVE_FALLOCATE)
+CHECK_FUNCTION_EXISTS(preadv HAVE_PREADV)
+INCLUDE(CheckIncludeFiles)
+CHECK_INCLUDE_FILES(linux/magic.h HAVE_MAGIC_H)
# This is a list of impala library dependencies. Individual libraries
# must not specify library dependencies in their own CMakeLists.txt file.
@@ -315,6 +329,7 @@ set (IMPALA_LINK_LIBS
Exprs
GlobalFlags
ImpalaThrift
+ kudu_util
Rpc
Runtime
Scheduling
@@ -366,7 +381,7 @@ set (IMPALA_DEPENDENCIES
glog
gflags
pprof
- breakpad
+ breakpad_client
hdfs
zlib
bzip2
@@ -471,6 +486,7 @@ add_subdirectory(src/codegen)
add_subdirectory(src/common)
add_subdirectory(src/exec)
add_subdirectory(src/exprs)
+add_subdirectory(src/kudu/util)
add_subdirectory(src/runtime)
add_subdirectory(src/scheduling)
add_subdirectory(src/statestore)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt
index 65edfc0..dd1e441 100644
--- a/be/src/common/CMakeLists.txt
+++ b/be/src/common/CMakeLists.txt
@@ -31,6 +31,7 @@ add_library(Common
init.cc
logging.cc
status.cc
+ kudu_version.cc
${VERSION_CC_GEN_OUTPUT}
)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/common/config.h.in
----------------------------------------------------------------------
diff --git a/be/src/common/config.h.in b/be/src/common/config.h.in
index fcae626..a33f2b9 100644
--- a/be/src/common/config.h.in
+++ b/be/src/common/config.h.in
@@ -19,8 +19,13 @@
#ifndef IMPALA_COMMON_CONFIG_H
#define IMPALA_COMMON_CONFIG_H
-/// This is a template that is populated by CMake with config information
+/// This is a template that is populated by CMake with config information.
+/// See be/CMakeLists.txt for the definition of the CMake symbols.
#cmakedefine HAVE_SCHED_GETCPU
+#cmakedefine IMPALA_HAVE_FALLOCATE
+#cmakedefine HAVE_PREADV
+#cmakedefine HAVE_PIPE2
+#cmakedefine HAVE_MAGIC_H
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/common/kudu_version.cc
----------------------------------------------------------------------
diff --git a/be/src/common/kudu_version.cc b/be/src/common/kudu_version.cc
new file mode 100644
index 0000000..3ffa009
--- /dev/null
+++ b/be/src/common/kudu_version.cc
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/util/version_info.h"
+
+#include "common/status.h"
+
+// Stubs to replace Kudu's VersionInfo implementation, which relies on generated version
+// files which are not part of our build process.
+
+namespace kudu {
+void VersionInfo::GetVersionInfoPB(VersionInfoPB* pb) {
+ DCHECK(false);
+}
+
+std::string VersionInfo::GetAllVersionInfo() {
+ DCHECK(false);
+ return "";
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/common/logging.cc
----------------------------------------------------------------------
diff --git a/be/src/common/logging.cc b/be/src/common/logging.cc
index 4ef1aeb..8025c9f 100644
--- a/be/src/common/logging.cc
+++ b/be/src/common/logging.cc
@@ -52,7 +52,9 @@ DECLARE_string(audit_event_log_dir);
using boost::uuids::random_generator;
+namespace {
bool logging_initialized = false;
+}
mutex logging_mutex;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/exec/kudu-util.h
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-util.h b/be/src/exec/kudu-util.h
index bc6d539..4503ce1 100644
--- a/be/src/exec/kudu-util.h
+++ b/be/src/exec/kudu-util.h
@@ -18,6 +18,9 @@
#ifndef IMPALA_UTIL_KUDU_UTIL_H_
#define IMPALA_UTIL_KUDU_UTIL_H_
+// TODO: Remove when toolchain callbacks.h properly defines ::tm.
+struct tm;
+
#include <kudu/client/callbacks.h>
#include <kudu/client/client.h>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/gutil
----------------------------------------------------------------------
diff --git a/be/src/kudu/gutil b/be/src/kudu/gutil
new file mode 120000
index 0000000..1d5bddc
--- /dev/null
+++ b/be/src/kudu/gutil
@@ -0,0 +1 @@
+../gutil
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/CMakeLists.txt b/be/src/kudu/util/CMakeLists.txt
index e4fff0c..4d8c5f5 100644
--- a/be/src/kudu/util/CMakeLists.txt
+++ b/be/src/kudu/util/CMakeLists.txt
@@ -130,7 +130,9 @@ set(UTIL_SRCS
debug/trace_event_impl.cc
debug/trace_event_impl_constants.cc
debug/trace_event_synthetic_delay.cc
- env.cc env_posix.cc env_util.cc
+ env.cc
+ env_posix.cc
+ env_util.cc
errno.cc
faststring.cc
failure_detector.cc
@@ -157,7 +159,7 @@ set(UTIL_SRCS
memory/overwrite.cc
mem_tracker.cc
metrics.cc
- minidump.cc
+ # minidump.cc
monotime.cc
mutex.cc
net/dns_resolver.cc
@@ -196,7 +198,9 @@ set(UTIL_SRCS
trace_metrics.cc
user.cc
url-coding.cc
- version_info.cc
+ # Remove from compilation, as it depends on generated method calls. Replaced by
+ # kudu_version.cc in Impala's common library.
+ # version_info.cc
zlib.cc
)
@@ -205,7 +209,7 @@ set(UTIL_SRCS
# optimized regardless of the default optimization options.
set_source_files_properties(memory/overwrite.cc PROPERTIES COMPILE_FLAGS "-O3")
-if(NOT APPLE)
+if(NOT NO_NVM_SUPPORT)
set(UTIL_SRCS
${UTIL_SRCS}
nvm_cache.cc)
@@ -229,7 +233,12 @@ if(NOT APPLE)
${UTIL_LIBS}
breakpad_client
dl
- rt
+ rt)
+endif()
+
+if (NOT NO_NVM_SUPPORT)
+ set(UTIL_LIBS
+ ${UTIL_LIBS}
vmem)
endif()
@@ -245,6 +254,11 @@ ADD_EXPORTABLE_LIBRARY(kudu_util
NONLINK_DEPS gen_version_info
EXPORTED_DEPS ${EXPORTED_UTIL_LIBS})
+if(NO_NVM_SUPPORT)
+ target_compile_definitions(kudu_util PRIVATE
+ NO_NVM_SUPPORT=1)
+endif()
+
#######################################
# kudu_util_compression
#######################################
@@ -276,7 +290,7 @@ if(NOT NO_TESTS)
gmock
kudu_util)
- if(NOT APPLE)
+ if(NOT NO_NVM_SUPPORT)
target_link_libraries(kudu_test_util
vmem)
endif()
@@ -318,7 +332,7 @@ endif()
#######################################
add_executable(protoc-gen-insertions protoc-gen-insertions.cc)
-target_link_libraries(protoc-gen-insertions gutil protobuf protoc ${KUDU_BASE_LIBS})
+target_link_libraries(protoc-gen-insertions gutil glog gflags protoc protobuf ${KUDU_BASE_LIBS})
#######################################
# Unit tests
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/cache.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/cache.cc b/be/src/kudu/util/cache.cc
index 4700fb3..c5954de 100644
--- a/be/src/kudu/util/cache.cc
+++ b/be/src/kudu/util/cache.cc
@@ -500,7 +500,7 @@ Cache* NewLRUCache(CacheType type, size_t capacity, const string& id) {
switch (type) {
case DRAM_CACHE:
return new ShardedLRUCache(capacity, id);
-#if !defined(__APPLE__)
+#if !defined(__APPLE__) && !defined(NO_NVM_SUPPORT)
case NVM_CACHE:
return NewLRUNvmCache(capacity, id);
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/compression/compression_codec.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/compression/compression_codec.cc b/be/src/kudu/util/compression/compression_codec.cc
index ee774cd..1a4e65b 100644
--- a/be/src/kudu/util/compression/compression_codec.cc
+++ b/be/src/kudu/util/compression/compression_codec.cc
@@ -21,8 +21,14 @@
#include <vector>
#include <glog/logging.h>
+#define LZ4_DISABLE_DEPRECATE_WARNINGS
#include <lz4.h>
#include <snappy-sinksource.h>
+
+// snappy.h redefines DISALLOW_COPY_AND_ASSIGN.
+#ifdef DISALLOW_COPY_AND_ASSIGN
+#undef DISALLOW_COPY_AND_ASSIGN
+#endif
#include <snappy.h>
#include <zlib.h>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/env.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/env.cc b/be/src/kudu/util/env.cc
index 1f6478c..76b68c9 100644
--- a/be/src/kudu/util/env.cc
+++ b/be/src/kudu/util/env.cc
@@ -7,6 +7,7 @@
#include <memory>
#include "kudu/util/faststring.h"
+#include "kudu/util/status.h"
using std::unique_ptr;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/env_posix.cc b/be/src/kudu/util/env_posix.cc
index a3998fc..6f9c9ff 100644
--- a/be/src/kudu/util/env_posix.cc
+++ b/be/src/kudu/util/env_posix.cc
@@ -53,14 +53,27 @@
#include "kudu/util/thread_restrictions.h"
#include "kudu/util/trace.h"
+// Compile-time checks for fallocate(), pread(), pwritev() etc.
+#include "common/config.h"
+
#if defined(__APPLE__)
#include <mach-o/dyld.h>
#include <sys/sysctl.h>
#else
#include <linux/falloc.h>
+
+// On RHEL5 this header causes compilation errors, due to issues with linux/types.h. There
+// is not an obvious good way to detect this issue at compile time, so instead use
+// IMPALA_HAVE_FALLOCATE as a proxy for RHEL5, and disable fiemap.h usage if fallocate()
+// is not available.
+#if defined(IMPALA_HAVE_FALLOCATE)
#include <linux/fiemap.h>
+#endif
+
#include <linux/fs.h>
+#if defined HAVE_MAGIC_H
#include <linux/magic.h>
+#endif
#include <sys/ioctl.h>
#include <sys/sysinfo.h>
#include <sys/vfs.h>
@@ -179,7 +192,13 @@ int fallocate(int fd, int mode, off_t offset, off_t len) {
}
return 0;
}
+#elif !defined(IMPALA_HAVE_FALLOCATE)
+int fallocate(int fd, int mode, off_t offset, off_t len) {
+ return EOPNOTSUPP;
+}
+#endif
+#if !defined(HAVE_PREADV)
// Simulates Linux's preadv API on OS X.
ssize_t preadv(int fd, const struct iovec* iovec, int count, off_t offset) {
ssize_t total_read_bytes = 0;
@@ -811,7 +830,7 @@ class PosixRWFile : public RWFile {
}
virtual Status GetExtentMap(ExtentMap* out) const OVERRIDE {
-#if !defined(__linux__)
+#if !defined(__linux__) || !defined(IMPALA_HAVE_FALLOCATE)
return Status::NotSupported("GetExtentMap not supported on this platform");
#else
TRACE_EVENT1("io", "PosixRWFile::GetExtentMap", "path", filename_);
@@ -1466,7 +1485,7 @@ class PosixEnv : public Env {
TRACE_EVENT0("io", "PosixEnv::IsOnExtFilesystem");
ThreadRestrictions::AssertIOAllowed();
-#ifdef __APPLE__
+#if defined __APPLE__ || !defined HAVE_MAGIC_H
*result = false;
#else
struct statfs buf;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/flags.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/flags.cc b/be/src/kudu/util/flags.cc
index 6b57f54..8cf2c39 100644
--- a/be/src/kudu/util/flags.cc
+++ b/be/src/kudu/util/flags.cc
@@ -60,10 +60,7 @@ DEFINE_bool(dump_metrics_json, false,
"by this binary.");
TAG_FLAG(dump_metrics_json, hidden);
-DEFINE_bool(enable_process_lifetime_heap_profiling, false, "Enables heap "
- "profiling for the lifetime of the process. Profile output will be stored in the "
- "directory specified by -heap_profile_path. Enabling this option will disable the "
- "on-demand/remote server profile handlers.");
+DECLARE_bool(enable_process_lifetime_heap_profiling);
TAG_FLAG(enable_process_lifetime_heap_profiling, stable);
TAG_FLAG(enable_process_lifetime_heap_profiling, advanced);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/kudu_export.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/kudu_export.h b/be/src/kudu/util/kudu_export.h
new file mode 100644
index 0000000..0d47c4d
--- /dev/null
+++ b/be/src/kudu/util/kudu_export.h
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef KUDU_EXPORT_H
+#define KUDU_EXPORT_H
+
+#ifdef KUDU_STATIC_DEFINE
+# define KUDU_EXPORT
+# define KUDU_NO_EXPORT
+#else
+# ifndef KUDU_EXPORT
+# ifdef kudu_client_exported_EXPORTS
+ /* We are building this library */
+# define KUDU_EXPORT __attribute__((visibility("default")))
+# else
+ /* We are using this library */
+# define KUDU_EXPORT __attribute__((visibility("default")))
+# endif
+# endif
+
+# ifndef KUDU_NO_EXPORT
+# define KUDU_NO_EXPORT __attribute__((visibility("hidden")))
+# endif
+#endif
+
+#ifndef KUDU_DEPRECATED
+# define KUDU_DEPRECATED __attribute__ ((__deprecated__))
+#endif
+
+#ifndef KUDU_DEPRECATED_EXPORT
+# define KUDU_DEPRECATED_EXPORT KUDU_EXPORT KUDU_DEPRECATED
+#endif
+
+#ifndef KUDU_DEPRECATED_NO_EXPORT
+# define KUDU_DEPRECATED_NO_EXPORT KUDU_NO_EXPORT KUDU_DEPRECATED
+#endif
+
+#if 0 /* DEFINE_NO_DEPRECATED */
+# ifndef KUDU_NO_DEPRECATED
+# define KUDU_NO_DEPRECATED
+# endif
+#endif
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/locks.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/locks.h b/be/src/kudu/util/locks.h
index 36d6984..db91559 100644
--- a/be/src/kudu/util/locks.h
+++ b/be/src/kudu/util/locks.h
@@ -178,7 +178,7 @@ class percpu_rwlock {
}
rw_spinlock &get_lock() {
-#if defined(__APPLE__) || defined(THREAD_SANITIZER)
+#if defined(__APPLE__) || defined(THREAD_SANITIZER) || !defined(HAVE_SCHED_GETCPU)
int cpu = 0;
#else
int cpu = sched_getcpu();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/logging.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/logging.cc b/be/src/kudu/util/logging.cc
index c62bfe7..5b2db1b 100644
--- a/be/src/kudu/util/logging.cc
+++ b/be/src/kudu/util/logging.cc
@@ -44,9 +44,7 @@
#include "kudu/util/signal.h"
#include "kudu/util/status.h"
-DEFINE_string(log_filename, "",
- "Prefix of log filename - "
- "full path is <log_dir>/<log_filename>.[INFO|WARN|ERROR|FATAL]");
+DECLARE_string(log_filename);
TAG_FLAG(log_filename, stable);
DEFINE_bool(log_async, true,
@@ -59,9 +57,7 @@ DEFINE_int32(log_async_buffer_bytes_per_level, 2 * 1024 * 1024,
"level. Only relevant when --log_async is enabled.");
TAG_FLAG(log_async_buffer_bytes_per_level, hidden);
-DEFINE_int32(max_log_files, 10,
- "Maximum number of log files to retain per severity level. The most recent "
- "log files are retained. If set to 0, all log files are retained.");
+DECLARE_int32(max_log_files);
TAG_FLAG(max_log_files, runtime);
TAG_FLAG(max_log_files, experimental);
@@ -271,7 +267,8 @@ void InitGoogleLoggingSafe(const char* arg) {
IgnoreSigPipe();
// For minidump support. Must be called before logging threads started.
- CHECK_OK(BlockSigUSR1());
+ // Disabled by Impala, which does not link Kudu's minidump library.
+ // CHECK_OK(BlockSigUSR1());
if (FLAGS_log_async) {
EnableAsyncLogging();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/minidump.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/minidump.cc b/be/src/kudu/util/minidump.cc
index c088ae7..a1be164 100644
--- a/be/src/kudu/util/minidump.cc
+++ b/be/src/kudu/util/minidump.cc
@@ -26,8 +26,8 @@
#include <string>
#if defined(__linux__)
-#include <breakpad/client/linux/handler/exception_handler.h>
-#include <breakpad/common/linux/linux_libc_support.h>
+#include <client/linux/handler/exception_handler.h>
+#include <common/linux/linux_libc_support.h>
#endif // defined(__linux__)
#include <gflags/gflags.h>
@@ -67,13 +67,7 @@ static bool ValidateMinidumpEnabled(const char* /*flagname*/, bool value) {
}
DEFINE_validator(enable_minidumps, &ValidateMinidumpEnabled);
-DEFINE_string(minidump_path, "minidumps", "Directory to write minidump files to. This "
- "can be either an absolute path or a path relative to --log_dir. Each daemon will "
- "create an additional sub-directory to prevent naming conflicts and to make it "
- "easier to identify a crashing daemon. Minidump files contain crash-related "
- "information in a compressed format. Minidumps will be written when a daemon exits "
- "unexpectedly, for example on an unhandled exception or signal, or when a "
- "SIGUSR1 signal is sent to the process. Cannot be set to an empty value.");
+DECLARE_string(minidump_path);
TAG_FLAG(minidump_path, evolving);
// The minidump path cannot be empty.
static bool ValidateMinidumpPath(const char* /*flagname*/, const string& value) {
@@ -81,14 +75,10 @@ static bool ValidateMinidumpPath(const char* /*flagname*/, const string& value)
}
DEFINE_validator(minidump_path, &ValidateMinidumpPath);
-DEFINE_int32(max_minidumps, 9, "Maximum number of minidump files to keep per daemon. "
- "Older files are removed first. Set to 0 to keep all minidump files.");
+DECLARE_int32(max_minidumps);
TAG_FLAG(max_minidumps, evolving);
-DEFINE_int32(minidump_size_limit_hint_kb, 20480, "Size limit hint for minidump files in "
- "KB. If a minidump exceeds this value, then breakpad will reduce the stack memory it "
- "collects for each thread from 8KB to 2KB. However it will always include the full "
- "stack memory for the first 20 threads, including the thread that crashed.");
+DECLARE_int32(minidump_size_limit_hint_kb);
TAG_FLAG(minidump_size_limit_hint_kb, advanced);
TAG_FLAG(minidump_size_limit_hint_kb, evolving);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/net/socket.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/net/socket.cc b/be/src/kudu/util/net/socket.cc
index e0bea14..cc0836b 100644
--- a/be/src/kudu/util/net/socket.cc
+++ b/be/src/kudu/util/net/socket.cc
@@ -120,7 +120,7 @@ bool Socket::IsTemporarySocketError(int err) {
return ((err == EAGAIN) || (err == EWOULDBLOCK) || (err == EINTR));
}
-#if defined(__linux__)
+#if defined(__linux__) && defined(SOCK_NONBLOCK) && defined(SOCK_CLOEXEC)
Status Socket::Init(int flags) {
int nonblocking_flag = (flags & FLAG_NONBLOCKING) ? SOCK_NONBLOCK : 0;
@@ -146,6 +146,7 @@ Status Socket::Init(int flags) {
RETURN_NOT_OK(SetNonBlocking(flags & FLAG_NONBLOCKING));
RETURN_NOT_OK(SetCloseOnExec());
+#if !defined(__linux__)
// Disable SIGPIPE.
int set = 1;
if (setsockopt(fd_, SOL_SOCKET, SO_NOSIGPIPE, &set, sizeof(set)) == -1) {
@@ -153,6 +154,7 @@ Status Socket::Init(int flags) {
return Status::NetworkError(std::string("failed to set SO_NOSIGPIPE: ") +
ErrnoToString(err), Slice(), err);
}
+#endif
return Status::OK();
}
@@ -332,7 +334,7 @@ Status Socket::Accept(Socket *new_conn, Sockaddr *remote, int flags) {
struct sockaddr_in addr;
socklen_t olen = sizeof(addr);
DCHECK_GE(fd_, 0);
-#if defined(__linux__)
+#if defined(__linux__) && defined(SOCK_NONBLOCK) && defined(SOCK_CLOEXEC)
int accept_flags = SOCK_CLOEXEC;
if (flags & FLAG_NONBLOCKING) {
accept_flags |= SOCK_NONBLOCK;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/be/src/kudu/util/subprocess.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/util/subprocess.cc b/be/src/kudu/util/subprocess.cc
index ec032cd..b072ed8 100644
--- a/be/src/kudu/util/subprocess.cc
+++ b/be/src/kudu/util/subprocess.cc
@@ -83,6 +83,11 @@ static const char* kProcSelfFd =
#define DIRENT dirent
#endif
+// Disable O_CLOEXEC if not available on very old kernels.
+#if !defined(O_CLOEXEC)
+#define O_CLOEXEC 0
+#endif
+
// Since opendir() calls malloc(), this must be called before fork().
// This function is not async-signal-safe.
Status OpenProcFdDir(DIR** dir) {
@@ -285,7 +290,7 @@ void Subprocess::DisableStdout() {
fd_state_[STDOUT_FILENO] = DISABLED;
}
-#if defined(__APPLE__)
+#if defined(__APPLE__) || !defined(HAVE_PIPE2)
static int pipe2(int pipefd[2], int flags) {
DCHECK_EQ(O_CLOEXEC, flags);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1cbfa423/bin/run_clang_tidy.sh
----------------------------------------------------------------------
diff --git a/bin/run_clang_tidy.sh b/bin/run_clang_tidy.sh
index 7d42597..ae389a9 100755
--- a/bin/run_clang_tidy.sh
+++ b/bin/run_clang_tidy.sh
@@ -17,8 +17,8 @@
# specific language governing permissions and limitations
# under the License.
-# This script runs clang-tidy on the backend, excluding gutil. clang-tidy finds bugs and
-# likely bugs.
+# This script runs clang-tidy on the backend, excluding gutil and kudu/. clang-tidy finds
+# bugs and likely bugs.
#
# To use this script, the toolchain must be installed and the toolchain environment
# variables must be set.
@@ -35,7 +35,7 @@ then
echo "WARNING: compile failed" >&2
fi
-DIRS=$(ls -d "${IMPALA_HOME}/be/src/"*/ | grep -v gutil | tr '\n' ' ')
+DIRS=$(ls -d "${IMPALA_HOME}/be/src/"*/ | grep -v gutil | grep -v kudu | tr '\n' ' ')
PIPE_DIRS=$(echo "${DIRS}" | tr ' ' '|')
# Reduce the concurrency to one less than the number of cores in the system. Note than