You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ab...@apache.org on 2021/11/16 20:46:00 UTC
[kudu] branch master updated (1754f51 -> c423172)
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from 1754f51 [client] fix MetaCacheEntry::Contains()
new cb69bb1 KUDU-1959 - Add test for /startup page for the Master
new dfb3ce9 [build] Introduce an env variable to indicate the thirdparty path
new c423172 [tests] Add iteration to test dir path
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 8 +++-
build-support/ccache-clang/clang | 3 +-
build-support/ccache-clang/clang++ | 3 +-
build-support/clang_tidy_gerrit.py | 6 +--
build-support/iwyu.py | 7 ++--
build-support/iwyu/iwyu_tool.py | 4 +-
build-support/jenkins/build-and-test.sh | 20 ++++++----
build-support/kudu_util.py | 5 +++
build-support/lint.sh | 3 +-
build-support/llvm-gcov-wrapper | 3 +-
.../mini-cluster/build_mini_cluster_binaries.sh | 7 ++--
docs/support/scripts/make_site.sh | 3 +-
src/kudu/clock/CMakeLists.txt | 4 +-
src/kudu/hms/CMakeLists.txt | 4 +-
src/kudu/master/master-test.cc | 45 ++++++++++++++++++++++
src/kudu/master/master.cc | 6 +--
src/kudu/postgres/CMakeLists.txt | 8 ++--
src/kudu/ranger/CMakeLists.txt | 4 +-
src/kudu/scripts/dump_breakpad_symbols.py | 11 +++++-
src/kudu/util/test_main.cc | 5 ++-
src/kudu/util/test_util.cc | 19 +++++++--
src/kudu/util/test_util.h | 4 ++
22 files changed, 135 insertions(+), 47 deletions(-)
[kudu] 01/03: KUDU-1959 - Add test for /startup page for the Master
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit cb69bb12eb07104c7d7c5333484534d997d65195
Author: Abhishek Chennaka <ac...@cloudera.com>
AuthorDate: Wed Oct 27 12:55:49 2021 -0400
KUDU-1959 - Add test for /startup page for the Master
This patch contains a test for the Kudu Master Startup Webpage.
We start up a mini master and validate the status of each startup step.
We also fix the startup timers for the mini master.
Change-Id: I823a59df215cfeb579f1593a63ef6133de37271c
Reviewed-on: http://gerrit.cloudera.org:8080/17989
Reviewed-by: Attila Bukor <ab...@apache.org>
Tested-by: Attila Bukor <ab...@apache.org>
---
src/kudu/master/master-test.cc | 45 ++++++++++++++++++++++++++++++++++++++++++
src/kudu/master/master.cc | 6 ++----
2 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 37493fa..289c12f 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -18,6 +18,7 @@
#include "kudu/master/master.h"
#include <algorithm>
+#include <atomic>
#include <cstdint>
#include <ctime>
#include <functional>
@@ -372,6 +373,50 @@ TEST_F(MasterTest, TestResetBlockCacheMetricsInSameProcess) {
});
}
+TEST_F(MasterTest, TestStartupWebPage) {
+ EasyCurl c;
+ faststring buf;
+ string addr = mini_master_->bound_http_addr().ToString();
+ mini_master_->Shutdown();
+ std::atomic<bool> run_status_reader = false;
+ thread read_startup_page([&] {
+ EasyCurl thread_c;
+ faststring thread_buf;
+ while (!run_status_reader) {
+ SleepFor(MonoDelta::FromMilliseconds(10));
+ if (!(thread_c.FetchURL(strings::Substitute("http://$0/startup", addr), &thread_buf)).ok()) {
+ continue;
+ }
+ ASSERT_STR_MATCHES(thread_buf.ToString(), "\"init_status\":(100|0)( |,)");
+ ASSERT_STR_MATCHES(thread_buf.ToString(), "\"read_filesystem_status\":(100|0)( |,)");
+ ASSERT_STR_MATCHES(thread_buf.ToString(), "\"read_instance_metadatafiles_status\""
+ ":(100|0)( |,)");
+ ASSERT_STR_MATCHES(thread_buf.ToString(), "\"read_data_directories_status\":"
+ "([0-9]|[1-9][0-9]|100)( |,)");
+ ASSERT_STR_MATCHES(thread_buf.ToString(), "\"initialize_master_catalog_status\":"
+ "([0-9]|[1-9][0-9]|100)( |,)");
+ ASSERT_STR_MATCHES(thread_buf.ToString(), "\"start_rpc_server_status\":(100|0)( |,)");
+ }
+ });
+ SCOPED_CLEANUP({
+ run_status_reader = true;
+ read_startup_page.join();
+ });
+
+ ASSERT_OK(mini_master_->Restart());
+ ASSERT_OK(mini_master_->WaitForCatalogManagerInit());
+ run_status_reader = true;
+
+ // After all the steps have been completed, ensure every startup step has 100 percent status
+ ASSERT_OK(c.FetchURL(strings::Substitute("http://$0/startup", addr), &buf));
+ ASSERT_STR_CONTAINS(buf.ToString(), "\"init_status\":100");
+ ASSERT_STR_CONTAINS(buf.ToString(), "\"read_filesystem_status\":100");
+ ASSERT_STR_CONTAINS(buf.ToString(), "\"read_instance_metadatafiles_status\":100");
+ ASSERT_STR_CONTAINS(buf.ToString(), "\"read_data_directories_status\":100");
+ ASSERT_STR_CONTAINS(buf.ToString(), "\"initialize_master_catalog_status\":100");
+ ASSERT_STR_CONTAINS(buf.ToString(), "\"start_rpc_server_status\":100");
+}
+
TEST_F(MasterTest, TestRegisterAndHeartbeat) {
const char* const kTsUUID = "my-ts-uuid";
diff --git a/src/kudu/master/master.cc b/src/kudu/master/master.cc
index ea401b8..6262a84 100644
--- a/src/kudu/master/master.cc
+++ b/src/kudu/master/master.cc
@@ -225,13 +225,9 @@ Status Master::Init() {
}
Status Master::Start() {
- Timer* init_master_catalog =
- startup_path_handler_->initialize_master_catalog_progress();
- init_master_catalog->Start();
RETURN_NOT_OK(StartAsync());
RETURN_NOT_OK(WaitForCatalogManagerInit());
google::FlushLogFiles(google::INFO); // Flush the startup messages.
- init_master_catalog->Stop();
return Status::OK();
}
@@ -281,11 +277,13 @@ Status Master::StartAsync() {
}
void Master::InitCatalogManagerTask() {
+ startup_path_handler_->initialize_master_catalog_progress()->Start();
Status s = InitCatalogManager();
if (!s.ok()) {
LOG(ERROR) << "Unable to init master catalog manager: " << s.ToString();
}
catalog_manager_init_status_.Set(s);
+ startup_path_handler_->initialize_master_catalog_progress()->Stop();
}
Status Master::InitCatalogManager() {
[kudu] 02/03: [build] Introduce an env variable to indicate the thirdparty path
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit dfb3ce9efe51ced39e3c83e229725e26abeb0136
Author: Yingchun Lai <ac...@gmail.com>
AuthorDate: Fri Oct 29 14:46:14 2021 +0800
[build] Introduce an env variable to indicate the thirdparty path
Kudu thirdparty will cost dozens of minutes to build, now introduce
an env variable THIRDPARTY_DIR to indicate the thirdparty path, it
can be a well built thirdparty libraries path.
With this feature, we can
- Share the same thirdparty path by different code paths, these code
paths are not needed to build thirdparty libraries repeatly
- Build an docker image with thirdparty libraries well built, then we
can use this image for building Kudu, CI, and etc.
Change-Id: I5e0bdf4faa44322622c48cacadf8e1165eccd38a
Reviewed-on: http://gerrit.cloudera.org:8080/18018
Tested-by: Kudu Jenkins
Reviewed-by: Attila Bukor <ab...@apache.org>
Reviewed-by: Andrew Wong <aw...@cloudera.com>
---
CMakeLists.txt | 8 ++++++--
build-support/ccache-clang/clang | 3 ++-
build-support/ccache-clang/clang++ | 3 ++-
build-support/clang_tidy_gerrit.py | 6 +++---
build-support/iwyu.py | 7 ++++---
build-support/iwyu/iwyu_tool.py | 4 +++-
build-support/jenkins/build-and-test.sh | 20 ++++++++++++--------
build-support/kudu_util.py | 5 +++++
build-support/lint.sh | 3 ++-
build-support/llvm-gcov-wrapper | 3 ++-
.../mini-cluster/build_mini_cluster_binaries.sh | 7 ++++---
docs/support/scripts/make_site.sh | 3 ++-
src/kudu/clock/CMakeLists.txt | 4 ++--
src/kudu/hms/CMakeLists.txt | 4 ++--
src/kudu/postgres/CMakeLists.txt | 8 ++++----
src/kudu/ranger/CMakeLists.txt | 4 ++--
src/kudu/scripts/dump_breakpad_symbols.py | 11 +++++++++--
17 files changed, 66 insertions(+), 37 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e276cf1..f3f657a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -90,8 +90,12 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
include(CMakeParseArguments)
set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build-support)
-set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty)
-set(THIRDPARTY_TOOLCHAIN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/clang-toolchain)
+if ("$ENV{THIRDPARTY_DIR}" STREQUAL "")
+ set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty)
+else()
+ set(THIRDPARTY_DIR $ENV{THIRDPARTY_DIR})
+endif()
+set(THIRDPARTY_TOOLCHAIN_DIR ${THIRDPARTY_DIR}/clang-toolchain)
set(THIRDPARTY_INSTALL_DIR ${THIRDPARTY_DIR}/installed)
set(THIRDPARTY_INSTALL_COMMON_DIR ${THIRDPARTY_INSTALL_DIR}/common)
set(THIRDPARTY_INSTALL_UNINSTRUMENTED_DIR ${THIRDPARTY_INSTALL_DIR}/uninstrumented)
diff --git a/build-support/ccache-clang/clang b/build-support/ccache-clang/clang
index a31b1ea..a56b27e 100755
--- a/build-support/ccache-clang/clang
+++ b/build-support/ccache-clang/clang
@@ -17,5 +17,6 @@
# under the License.
ROOT=$(dirname $BASH_SOURCE)/../..
-CLANG=$ROOT/thirdparty/clang-toolchain/bin/clang
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty}
+CLANG=${THIRDPARTY_DIR}/clang-toolchain/bin/clang
CCACHE_CPP2=yes exec ccache $CLANG -Qunused-arguments `test -t 2 && echo -fcolor-diagnostics` "$@"
diff --git a/build-support/ccache-clang/clang++ b/build-support/ccache-clang/clang++
index 9cf7f90..fe1b7d8 100755
--- a/build-support/ccache-clang/clang++
+++ b/build-support/ccache-clang/clang++
@@ -17,5 +17,6 @@
# under the License.
ROOT=$(dirname $BASH_SOURCE)/../..
-CLANG=$ROOT/thirdparty/clang-toolchain/bin/clang++
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty}
+CLANG=${THIRDPARTY_DIR}/clang-toolchain/bin/clang++
CCACHE_CPP2=yes exec ccache $CLANG -Qunused-arguments `test -t 2 && echo -fcolor-diagnostics` "$@"
diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py
index 35f32f6..aba74ae 100755
--- a/build-support/clang_tidy_gerrit.py
+++ b/build-support/clang_tidy_gerrit.py
@@ -33,16 +33,16 @@ import sys
import unittest
import tempfile
-from kudu_util import init_logging
+from kudu_util import init_logging, get_thirdparty_dir
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
BUILD_PATH = os.path.join(ROOT, "build", "latest")
CLANG_TIDY_DIFF = os.path.join(
- ROOT, "thirdparty/installed/uninstrumented/share/clang/clang-tidy-diff.py")
+ get_thirdparty_dir(), "installed/uninstrumented/share/clang/clang-tidy-diff.py")
CLANG_TIDY = os.path.join(
- ROOT, "thirdparty/clang-toolchain/bin/clang-tidy")
+ get_thirdparty_dir(), "clang-toolchain/bin/clang-tidy")
GERRIT_USER = 'tidybot'
GERRIT_PASSWORD = os.getenv('TIDYBOT_PASSWORD')
diff --git a/build-support/iwyu.py b/build-support/iwyu.py
index ce965d5..4bb8f28 100755
--- a/build-support/iwyu.py
+++ b/build-support/iwyu.py
@@ -30,7 +30,8 @@ import re
import subprocess
import sys
-from kudu_util import get_upstream_commit, check_output, ROOT, Colors, init_logging
+from kudu_util import get_upstream_commit, check_output, ROOT, Colors, \
+ init_logging, get_thirdparty_dir
import iwyu.fix_includes
from iwyu.fix_includes import ParseAndMergeIWYUOutput
@@ -44,7 +45,7 @@ script in order to fix any reported issues.
"""
_MAPPINGS_DIR = os.path.join(ROOT, "build-support/iwyu/mappings/")
-_TOOLCHAIN_DIR = os.path.join(ROOT, "thirdparty/clang-toolchain/bin")
+_TOOLCHAIN_DIR = os.path.join(get_thirdparty_dir(), "clang-toolchain/bin")
_IWYU_TOOL = os.path.join(ROOT, "build-support/iwyu/iwyu_tool.py")
# Matches source files that we should run on.
@@ -129,7 +130,7 @@ def _relativize_paths(paths):
def _get_thirdparty_include_dirs():
- return glob.glob(os.path.join(ROOT, "thirdparty", "installed", "*", "include"))
+ return glob.glob(os.path.join(get_thirdparty_dir(), "installed", "*", "include"))
def _get_fixer_flags(flags):
diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py
index d2a15ea..8eb7d44 100755
--- a/build-support/iwyu/iwyu_tool.py
+++ b/build-support/iwyu/iwyu_tool.py
@@ -85,6 +85,8 @@ import re
import subprocess
import sys
import unittest
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from kudu_util import get_thirdparty_dir
def iwyu_formatter(output):
@@ -232,7 +234,7 @@ def workaround_add_libcpp(build_dir, command):
#
# Some sanitizer builds (like TSAN) already include libc++; if the command
# already has a flag including libc++, we don't want to add it again.
- path_to_libcpp_prefix = os.path.join(build_dir, "..", "..", "thirdparty", "installed")
+ path_to_libcpp_prefix = os.path.join(get_thirdparty_dir(), "installed")
path_to_libcpp_suffix = os.path.join("include", "c++", "v1")
tp_pattern = os.path.join(path_to_libcpp_prefix, "*", path_to_libcpp_suffix)
found = False
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index b2ae527..a76476a 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -149,12 +149,17 @@ if [ "$BUILD_TYPE" = "TSAN" ]; then
export KUDU_USE_TSAN=1
fi
+SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
+BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER
+
export KUDU_FLAKY_TEST_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
export KUDU_ALLOW_SLOW_TESTS=${KUDU_ALLOW_SLOW_TESTS:-$DEFAULT_ALLOW_SLOW_TESTS}
export KUDU_COMPRESS_TEST_OUTPUT=${KUDU_COMPRESS_TEST_OUTPUT:-1}
export TEST_TMPDIR=${TEST_TMPDIR:-/tmp/kudutest-$UID}
export PARALLEL=${PARALLEL:-$(getconf _NPROCESSORS_ONLN)}
export PARALLEL_TESTS=${PARALLEL_TESTS:-$PARALLEL}
+export THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty}
+
BUILD_JAVA=${BUILD_JAVA:-1}
BUILD_GRADLE=${BUILD_GRADLE:-1}
BUILD_PYTHON=${BUILD_PYTHON:-1}
@@ -168,9 +173,6 @@ if [ ! -w "$TEST_TMPDIR" ]; then
exit 1
fi
-SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
-BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER
-
# Remove testing artifacts from the previous run before we do anything
# else. Otherwise, if we fail during the "build" step, Jenkins will
# archive the test logs from the previous run, thinking they came from
@@ -312,16 +314,18 @@ if [ "$BUILD_TYPE" = "TSAN" ]; then
THIRDPARTY_TYPE=tsan
fi
-# The settings for PARALLEL (see above) also affects the thirdparty build.
-$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE
+if [ ! -n "${NO_REBUILD_THIRDPARTY}" ]; then
+ # The settings for PARALLEL (see above) also affects the thirdparty build.
+ $SOURCE_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_DIR/build-if-necessary.sh $THIRDPARTY_TYPE
+fi
-THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
+THIRDPARTY_BIN=$THIRDPARTY_DIR/installed/common/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof
if which ccache >/dev/null ; then
CLANG=$(pwd)/build-support/ccache-clang/clang
else
- CLANG=$(pwd)/thirdparty/clang-toolchain/bin/clang
+ CLANG=$THIRDPARTY_DIR/clang-toolchain/bin/clang
fi
# Make sure we use JDK8
@@ -517,7 +521,7 @@ if [ "$DO_COVERAGE" == "1" ]; then
echo
echo Generating coverage report...
echo ------------------------------------------------------------
- if ! $SOURCE_ROOT/thirdparty/installed/common/bin/gcovr \
+ if ! $THIRDPARTY_DIR/installed/common/bin/gcovr \
-r $SOURCE_ROOT \
--gcov-filter='.*src#kudu.*' \
--gcov-executable=$SOURCE_ROOT/build-support/llvm-gcov-wrapper \
diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
index 15d1059..df079f2 100644
--- a/build-support/kudu_util.py
+++ b/build-support/kudu_util.py
@@ -114,3 +114,8 @@ def get_my_email():
""" Return the email address in the user's git config. """
return check_output(['git', 'config', '--get',
'user.email']).strip().decode('utf-8')
+
+
+def get_thirdparty_dir():
+ env = os.environ.copy()
+ return env['THIRDPARTY_DIR'] if env.has_key('THIRDPARTY_DIR') else os.path.join(ROOT, "thirdparty")
diff --git a/build-support/lint.sh b/build-support/lint.sh
index f93168d..7f10f83 100755
--- a/build-support/lint.sh
+++ b/build-support/lint.sh
@@ -50,7 +50,8 @@ fi
cpplint_filter="+runtime/broken_libstdcpp_regex,-whitespace/comments,-readability/todo,-readability/inheritance,-build/header_guard,-build/include_order,-legal/copyright,-build/c++11,-readability/nolint"
cd $ROOT
-$ROOT/thirdparty/installed/common/bin/cpplint.py \
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty}
+${THIRDPARTY_DIR}/installed/common/bin/cpplint.py \
--verbose=4 \
--filter=$cpplint_filter \
$FILES 2>&1 | grep -v 'Done processing' | tee $TMP
diff --git a/build-support/llvm-gcov-wrapper b/build-support/llvm-gcov-wrapper
index 0b24685..42b4ead 100755
--- a/build-support/llvm-gcov-wrapper
+++ b/build-support/llvm-gcov-wrapper
@@ -20,4 +20,5 @@
# gcovr can call it.
SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/..; pwd)
-exec $SOURCE_ROOT/thirdparty/clang-toolchain/bin/llvm-cov gcov "$@"
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty}
+exec $THIRDPARTY_DIR/clang-toolchain/bin/llvm-cov gcov "$@"
diff --git a/build-support/mini-cluster/build_mini_cluster_binaries.sh b/build-support/mini-cluster/build_mini_cluster_binaries.sh
index da8d630..fe966f5 100755
--- a/build-support/mini-cluster/build_mini_cluster_binaries.sh
+++ b/build-support/mini-cluster/build_mini_cluster_binaries.sh
@@ -84,13 +84,14 @@ if [ -d "$BUILD_ROOT" ]; then
fi
fi
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$BASE_DIR/thirdparty}
cd $SOURCE_ROOT
if [ -n "$NO_REBUILD_THIRDPARTY" ]; then
echo Skipping thirdparty because NO_REBUILD_THIRDPARTY is not empty
else
echo Building thirdparty... >&2
$SOURCE_ROOT/build-support/enable_devtoolset.sh \
- ./thirdparty/build-if-necessary.sh
+ $THIRDPARTY_DIR/build-if-necessary.sh
fi
mkdir -p $BUILD_ROOT
@@ -121,7 +122,7 @@ rm -rf CMakeCache.txt CMakeFiles
# with the mini-cluster.
echo Configuring Kudu... >&2
$SOURCE_ROOT/build-support/enable_devtoolset.sh \
- $SOURCE_ROOT/thirdparty/installed/common/bin/cmake ../.. \
+ $THIRDPARTY_DIR/installed/common/bin/cmake ../.. \
-DNO_TESTS=1 -DNO_CHRONY=1 \
-DCMAKE_BUILD_TYPE=RELEASE -DKUDU_LINK=dynamic $EXTRA_CMAKE_FLAGS
@@ -191,7 +192,7 @@ if [ $MACOS ]; then
JAR_LICENSE=$MINI_CLUSTER_SRCDIR/LICENSE-BINARY-JAR-OSX.txt
fi
cat $SOURCE_ROOT/LICENSE.txt \
- $SOURCE_ROOT/thirdparty/LICENSE.txt \
+ $THIRDPARTY_DIR/LICENSE.txt \
$JAR_LICENSE \
> $ARTIFACT_NAME/LICENSE.txt
diff --git a/docs/support/scripts/make_site.sh b/docs/support/scripts/make_site.sh
index b01e642..3adbe6f 100755
--- a/docs/support/scripts/make_site.sh
+++ b/docs/support/scripts/make_site.sh
@@ -73,8 +73,9 @@ if [ -z "$GIT_REMOTE" ]; then
usage
fi
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty}
# Build Kudu thirdparty
-$SOURCE_ROOT/build-support/enable_devtoolset.sh $SOURCE_ROOT/thirdparty/build-if-necessary.sh
+$SOURCE_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_DIR/build-if-necessary.sh
echo "Successfully built third-party dependencies."
# Build the binaries so we can auto-generate the command-line references
diff --git a/src/kudu/clock/CMakeLists.txt b/src/kudu/clock/CMakeLists.txt
index b57d30f..0c20fea 100644
--- a/src/kudu/clock/CMakeLists.txt
+++ b/src/kudu/clock/CMakeLists.txt
@@ -45,10 +45,10 @@ if (NOT NO_CHRONY)
# Link the chrony binaries so that they can be found via
# MiniChronyd::GetPath in MiniChronyd::Start.
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/bin/chronyc"
+ "${THIRDPARTY_DIR}/installed/common/bin/chronyc"
"${EXECUTABLE_OUTPUT_PATH}/chronyc")
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/sbin/chronyd"
+ "${THIRDPARTY_DIR}/installed/common/sbin/chronyd"
"${EXECUTABLE_OUTPUT_PATH}/chronyd")
set(MINI_CHRONYD_SRCS test/mini_chronyd.cc)
diff --git a/src/kudu/hms/CMakeLists.txt b/src/kudu/hms/CMakeLists.txt
index ef1171b..e016d84 100644
--- a/src/kudu/hms/CMakeLists.txt
+++ b/src/kudu/hms/CMakeLists.txt
@@ -54,10 +54,10 @@ target_link_libraries(kudu_hms ${HMS_DEPS})
# Link the home directories, so that they can be found via
# test_util::FindHomeDir in MiniHms::Start.
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hive"
+ "${THIRDPARTY_DIR}/installed/common/opt/hive"
"${EXECUTABLE_OUTPUT_PATH}/hive-home")
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hadoop"
+ "${THIRDPARTY_DIR}/installed/common/opt/hadoop"
"${EXECUTABLE_OUTPUT_PATH}/hadoop-home")
execute_process(COMMAND ln -nsf
"${JAVA_HOME}"
diff --git a/src/kudu/postgres/CMakeLists.txt b/src/kudu/postgres/CMakeLists.txt
index e9c97a5..6daa30f 100644
--- a/src/kudu/postgres/CMakeLists.txt
+++ b/src/kudu/postgres/CMakeLists.txt
@@ -20,16 +20,16 @@
#######################################
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/bin"
+ "${THIRDPARTY_DIR}/installed/common/bin"
"${EXECUTABLE_OUTPUT_PATH}/postgres")
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/lib"
+ "${THIRDPARTY_DIR}/installed/common/lib"
"${EXECUTABLE_OUTPUT_PATH}/postgres-lib")
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/share/postgresql"
+ "${THIRDPARTY_DIR}/installed/common/share/postgresql"
"${EXECUTABLE_OUTPUT_PATH}/postgres-share")
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/jdbc/postgresql.jar"
+ "${THIRDPARTY_DIR}/installed/common/opt/jdbc/postgresql.jar"
"${EXECUTABLE_OUTPUT_PATH}/postgresql.jar")
add_library(mini_postgres
diff --git a/src/kudu/ranger/CMakeLists.txt b/src/kudu/ranger/CMakeLists.txt
index a52040b..0c6241f 100644
--- a/src/kudu/ranger/CMakeLists.txt
+++ b/src/kudu/ranger/CMakeLists.txt
@@ -52,10 +52,10 @@ target_link_libraries(kudu_ranger ${RANGER_DEPS})
#######################################
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/ranger"
+ "${THIRDPARTY_DIR}/installed/common/opt/ranger"
"${EXECUTABLE_OUTPUT_PATH}/ranger-home")
execute_process(COMMAND ln -nsf
- "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hadoop"
+ "${THIRDPARTY_DIR}/installed/common/opt/hadoop"
"${EXECUTABLE_OUTPUT_PATH}/hadoop-home")
execute_process(COMMAND ln -nsf
"${JAVA_HOME}"
diff --git a/src/kudu/scripts/dump_breakpad_symbols.py b/src/kudu/scripts/dump_breakpad_symbols.py
index bdb2921..f18246c 100644
--- a/src/kudu/scripts/dump_breakpad_symbols.py
+++ b/src/kudu/scripts/dump_breakpad_symbols.py
@@ -56,7 +56,7 @@
# python dump_breakpad_symbols.py -d /tmp/syms -b kudu/build/debug
#
# * After, to process a minidump file, use the 'minidump_stackwalk' tool:
-# $KUDU_HOME/thirdparty/installed/uninstrumented/bin/minidump_stackwalk \
+# $THIRDPARTY_DIR/installed/uninstrumented/bin/minidump_stackwalk \
# /tmp/kudu-minidumps/kudu-tserver/03c0ee26-bfd1-cf3e-43fa49ca-1a6aae25.dmp /tmp/syms
#
# For more information, see the getting started docs at
@@ -86,6 +86,12 @@ def die(msg=''):
sys.exit(1)
+def get_thirdparty_dir():
+ env = os.environ.copy()
+ return env['THIRDPARTY_DIR'] if env.has_key('THIRDPARTY_DIR') else \
+ os.path.join(ROOT, "thirdparty")
+
+
def find_dump_syms_binary():
"""Locate the 'dump_syms' binary from Breakpad.
@@ -98,7 +104,8 @@ def find_dump_syms_binary():
if not os.path.isdir(kudu_home):
die('Could not find KUDU_HOME directory')
# TODO: Use dump_syms_mac if on macOS.
- dump_syms = os.path.join(kudu_home, 'thirdparty', 'installed', 'uninstrumented', 'bin', 'dump_syms')
+ dump_syms = os.path.join(get_thirdparty_dir(),
+ 'installed', 'uninstrumented', 'bin', 'dump_syms')
if not os.path.isfile(dump_syms):
die('Could not find dump_syms executable at %s' % dump_syms)
return dump_syms
[kudu] 03/03: [tests] Add iteration to test dir path
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit c423172ca60383a63b3e5ca00f767ccc11001180
Author: Attila Bukor <ab...@apache.org>
AuthorDate: Tue Nov 16 18:07:41 2021 +0100
[tests] Add iteration to test dir path
GTest supports running tests repeatedly by setting a "--gtest_repeat"
flag, which can be incredibly useful to debug flaky tests when using
together with "--gtest_break_on_failure" in gdb.
Unfortunately, most of our tests expect an empty data directory, but the
test directory was the same across repeated runs without being emptied.
Adding the iteration number to the end of the directory solves this
problem and --gtest_repeat can be used safely.
Change-Id: I175756e6f758246f2234eae3a0ad99a544f80026
Reviewed-on: http://gerrit.cloudera.org:8080/18030
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Tested-by: Attila Bukor <ab...@apache.org>
---
src/kudu/util/test_main.cc | 5 +++--
src/kudu/util/test_util.cc | 19 +++++++++++++++----
src/kudu/util/test_util.h | 4 ++++
3 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/src/kudu/util/test_main.cc b/src/kudu/util/test_main.cc
index c75e5ae..ccdc418 100644
--- a/src/kudu/util/test_main.cc
+++ b/src/kudu/util/test_main.cc
@@ -103,7 +103,8 @@ int main(int argc, char **argv) {
// cover our bases and call it here too.
kudu::KuduTest::OverrideKrb5Environment();
- int ret = RUN_ALL_TESTS();
+ testing::TestEventListeners& listeners = testing::UnitTest::GetInstance()->listeners();
+ listeners.Append(new kudu::KuduTestEventListener());
- return ret;
+ return RUN_ALL_TESTS();
}
diff --git a/src/kudu/util/test_util.cc b/src/kudu/util/test_util.cc
index dd6eb1b..a01fbb0 100644
--- a/src/kudu/util/test_util.cc
+++ b/src/kudu/util/test_util.cc
@@ -71,6 +71,10 @@ using std::string;
using std::vector;
using strings::Substitute;
+namespace {
+int testIteration = 0;
+} // namespace
+
namespace kudu {
const char* kInvalidPath = "/dev/invalid-path-for-kudu-tests";
@@ -86,6 +90,11 @@ static const uint64_t kTestBeganAtMicros = Env::Default()->NowMicros();
// This can be checked using the 'IsGTest()' function from test_util_prod.cc.
bool g_is_gtest = true;
+void KuduTestEventListener::OnTestIterationStart(const testing::UnitTest& /*unit_test*/,
+ int iteration) {
+ testIteration = iteration;
+}
+
///////////////////////////////////////////////////
// KuduTest
///////////////////////////////////////////////////
@@ -130,7 +139,7 @@ KuduTest::KuduTest()
// If the TEST_TMPDIR variable has been set, then glog will automatically use that
// as its default log directory. We would prefer that the default log directory
// instead be the test-case-specific subdirectory.
- FLAGS_log_dir = GetTestDataDirectory();
+ FLAGS_log_dir = test_dir_;
}
KuduTest::~KuduTest() {
@@ -223,8 +232,9 @@ string GetTestDataDirectory() {
// The directory name includes some strings for specific reasons:
// - program name: identifies the directory to the test invoker
// - timestamp and pid: disambiguates with prior runs of the same test
+ // - iteration: identifies the iteration when using --gtest_repeat
//
- // e.g. "env-test.TestEnv.TestReadFully.1409169025392361-23600"
+ // e.g. "env-test.TestEnv.TestReadFully.1409169025392361-23600-0"
//
// If the test is sharded, the shard index is also included so that the test
// invoker can more easily identify all directories belonging to each shard.
@@ -233,13 +243,14 @@ string GetTestDataDirectory() {
if (shard_index && shard_index[0] != '\0') {
shard_index_infix = Substitute("$0.", shard_index);
}
- dir += Substitute("/$0.$1$2.$3.$4-$5",
+ dir += Substitute("/$0.$1$2.$3.$4-$5-$6",
StringReplace(google::ProgramInvocationShortName(), "/", "_", true),
shard_index_infix,
StringReplace(test_info->test_case_name(), "/", "_", true),
StringReplace(test_info->name(), "/", "_", true),
kTestBeganAtMicros,
- getpid());
+ getpid(),
+ testIteration);
Status s = Env::Default()->CreateDir(dir);
CHECK(s.IsAlreadyPresent() || s.ok())
<< "Could not create directory " << dir << ": " << s.ToString();
diff --git a/src/kudu/util/test_util.h b/src/kudu/util/test_util.h
index d9150d7..47bdd0e 100644
--- a/src/kudu/util/test_util.h
+++ b/src/kudu/util/test_util.h
@@ -55,6 +55,10 @@ class Status;
extern const char* kInvalidPath;
+class KuduTestEventListener : public ::testing::EmptyTestEventListener {
+ void OnTestIterationStart(const testing::UnitTest& unit_test, int iteration) override;
+};
+
class KuduTest : public ::testing::Test {
public:
KuduTest();