You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2018/05/10 22:53:45 UTC
[1/3] kudu git commit: [tools] Add ids to tables and tablets;
table name to tablets
Repository: kudu
Updated Branches:
refs/heads/master 3f43c03a1 -> 3855328b7
[tools] Add ids to tables and tablets; table name to tablets
This adds tablet id, table id, and table name to KsckTabletSummary,
and table id to KsckTableSummary.
This will be tested in the follow-up JSON formatting patch.
Change-Id: I19cd76a4c4c59c28930a44c0593021039903bec1
Reviewed-on: http://gerrit.cloudera.org:8080/10369
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Tested-by: Will Berkeley <wd...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/525943fd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/525943fd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/525943fd
Branch: refs/heads/master
Commit: 525943fd189acc20a3199f84431d62a776dd8f30
Parents: 3f43c03
Author: Will Berkeley <wd...@apache.org>
Authored: Sun May 6 12:56:41 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Thu May 10 20:07:41 2018 +0000
----------------------------------------------------------------------
src/kudu/tools/ksck-test.cc | 5 +++--
src/kudu/tools/ksck.cc | 6 +++++-
src/kudu/tools/ksck.h | 12 ++++++++++--
src/kudu/tools/ksck_remote.cc | 3 ++-
src/kudu/tools/ksck_results.h | 4 ++++
5 files changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck-test.cc b/src/kudu/tools/ksck-test.cc
index f4011cc..18d28ef 100644
--- a/src/kudu/tools/ksck-test.cc
+++ b/src/kudu/tools/ksck-test.cc
@@ -278,8 +278,9 @@ class KsckTest : public KuduTest {
table->set_tablets({ tablet });
}
- shared_ptr<KsckTable> CreateAndAddTable(const string& name, int num_replicas) {
- shared_ptr<KsckTable> table(new KsckTable(name, Schema(), num_replicas));
+ shared_ptr<KsckTable> CreateAndAddTable(const string& id_and_name, int num_replicas) {
+ shared_ptr<KsckTable> table(new KsckTable(id_and_name, id_and_name,
+ Schema(), num_replicas));
cluster_->tables_.push_back(table);
return table;
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index 191838b..70a1b64 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -716,10 +716,11 @@ bool Ksck::VerifyTable(const shared_ptr<KsckTable>& table) {
}
KsckTableSummary ts;
+ ts.id = table->id();
+ ts.name = table->name();
ts.replication_factor = table->num_replicas();
VLOG(1) << Substitute("Verifying $0 tablet(s) for table $1 configured with num_replicas = $2",
tablets.size(), table->name(), table->num_replicas());
- ts.name = table->name();
for (const auto& tablet : tablets) {
auto tablet_result = VerifyTablet(tablet, table->num_replicas());
switch (tablet_result) {
@@ -876,6 +877,9 @@ KsckCheckResult Ksck::VerifyTablet(const shared_ptr<KsckTablet>& tablet,
}
KsckTabletSummary tablet_summary;
+ tablet_summary.id = tablet->id();
+ tablet_summary.table_id = tablet->table()->id();
+ tablet_summary.table_name = tablet->table()->name();
tablet_summary.result = result;
tablet_summary.status = status;
tablet_summary.master_cstate = std::move(master_config);
http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.h b/src/kudu/tools/ksck.h
index edae81b..1c596e4 100644
--- a/src/kudu/tools/ksck.h
+++ b/src/kudu/tools/ksck.h
@@ -138,8 +138,15 @@ class KsckTablet {
// Representation of a table. Composed of tablets.
class KsckTable {
public:
- KsckTable(std::string name, const Schema& schema, int num_replicas)
- : name_(std::move(name)), schema_(schema), num_replicas_(num_replicas) {}
+ KsckTable(std::string id, std::string name, const Schema& schema, int num_replicas)
+ : id_(std::move(id)),
+ name_(std::move(name)),
+ schema_(schema),
+ num_replicas_(num_replicas) {}
+
+ const std::string& id() const {
+ return id_;
+ }
const std::string& name() const {
return name_;
@@ -162,6 +169,7 @@ class KsckTable {
}
private:
+ const std::string id_;
const std::string name_;
const Schema schema_;
const int num_replicas_;
http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck_remote.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_remote.cc b/src/kudu/tools/ksck_remote.cc
index d24f8c7..017321e 100644
--- a/src/kudu/tools/ksck_remote.cc
+++ b/src/kudu/tools/ksck_remote.cc
@@ -426,7 +426,8 @@ Status RemoteKsckCluster::RetrieveTablesList() {
client::sp::shared_ptr<KuduTable> t;
RETURN_NOT_OK(client_->OpenTable(n, &t));
- shared_ptr<KsckTable> table(new KsckTable(n,
+ shared_ptr<KsckTable> table(new KsckTable(t->id(),
+ n,
*t->schema().schema_,
t->num_replicas()));
tables_temp.push_back(table);
http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck_results.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.h b/src/kudu/tools/ksck_results.h
index 1e60be2..55b8920 100644
--- a/src/kudu/tools/ksck_results.h
+++ b/src/kudu/tools/ksck_results.h
@@ -142,6 +142,7 @@ struct KsckServerHealthSummary {
// A summary of the state of a table.
struct KsckTableSummary {
+ std::string id;
std::string name;
int replication_factor = 0;
int healthy_tablets = 0;
@@ -201,6 +202,9 @@ struct KsckReplicaSummary {
// A summary of the state of a tablet.
struct KsckTabletSummary {
+ std::string id;
+ std::string table_id;
+ std::string table_name;
KsckCheckResult result;
std::string status;
KsckConsensusState master_cstate;
[2/3] kudu git commit: Fix int overflow GetClockTimeMicros() on macOS
Posted by da...@apache.org.
Fix int overflow GetClockTimeMicros() on macOS
On macOS mach_timespec_t.tv_sec is only 4 bytes and we
were converting to micros before moving to a bigger var.
This would cause all the wall times obtained on a mac (through
GetClockTimeMicros() to be wrong.
This was likely the cause of KUDU-2435 and KUDU-2408 too, since
the time would easily wrap, causing us to a update the clock with
a value that was seemingly from the future.
Posix just requires it to be an integer
(see: https://en.wikipedia.org/w/index.php?title=Time_t&oldid=450752800)
so also fixed it on the non-macOS path.
Testing this is likely not worth it since the only real change
was on macOS where the overlow doesn't happen anymore.
Change-Id: Ie0eaa548f61352be529755a732566613cfa72098
Reviewed-on: http://gerrit.cloudera.org:8080/10371
Reviewed-by: Todd Lipcon <to...@apache.org>
Reviewed-by: Dan Burkert <da...@apache.org>
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/07d6b5f2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/07d6b5f2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/07d6b5f2
Branch: refs/heads/master
Commit: 07d6b5f2ba384e10288d5926af93933f0a74185d
Parents: 525943f
Author: David Alves <dr...@apache.org>
Authored: Thu May 10 13:52:14 2018 -0700
Committer: David Ribeiro Alves <da...@gmail.com>
Committed: Thu May 10 22:38:23 2018 +0000
----------------------------------------------------------------------
src/kudu/gutil/walltime.h | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/07d6b5f2/src/kudu/gutil/walltime.h
----------------------------------------------------------------------
diff --git a/src/kudu/gutil/walltime.h b/src/kudu/gutil/walltime.h
index e9cab67..4b15f42 100644
--- a/src/kudu/gutil/walltime.h
+++ b/src/kudu/gutil/walltime.h
@@ -81,7 +81,12 @@ inline void GetCurrentTime(mach_timespec_t* ts) {
inline MicrosecondsInt64 GetCurrentTimeMicros() {
mach_timespec_t ts;
GetCurrentTime(&ts);
- return ts.tv_sec * 1000000 + ts.tv_nsec / 1000;
+ // 'tv_sec' is just 4 bytes on macOS, need to be careful not
+ // to convert to nanos until we've moved to a larger int.
+ MicrosecondsInt64 micros_from_secs = ts.tv_sec;
+ micros_from_secs *= 1000 * 1000;
+ micros_from_secs += ts.tv_nsec / 1000;
+ return micros_from_secs;
}
inline int64_t GetMonoTimeNanos() {
@@ -130,7 +135,13 @@ inline MicrosecondsInt64 GetThreadCpuTimeMicros() {
inline MicrosecondsInt64 GetClockTimeMicros(clockid_t clock) {
timespec ts;
clock_gettime(clock, &ts);
- return ts.tv_sec * 1000000 + ts.tv_nsec / 1000;
+ // 'tv_sec' is usually 8 bytes, but the spec says it only
+ // needs to be 'a signed int'. Moved to a 64 bit var before
+ // converting to micros to be safe.
+ MicrosecondsInt64 micros_from_secs = ts.tv_sec;
+ micros_from_secs *= 1000 * 1000;
+ micros_from_secs += ts.tv_nsec / 1000;
+ return micros_from_secs;
}
#endif // defined(__APPLE__)
[3/3] kudu git commit: thirdparty: tweak clang compiler flags
Posted by da...@apache.org.
thirdparty: tweak clang compiler flags
Prior to this small tweak the thirdparty clang build output thousands of
warnings similar to this one when compiling with clang:
Building CXX object lib/Analysis/CMakeFiles/LLVMAnalysis.dir/LazyValueInfo.cpp.o
clang: warning: -Wl,-rpath,/Users/dan/src/cpp/kudu/thirdparty/installed/uninstrumented/lib: 'linker' input unused [-Wunused-command-line-argument]
clang: warning: argument unused during compilation: '-L/Users/dan/src/cpp/kudu/thirdparty/installed/uninstrumented/lib' [-Wunused-command-line-argument]
Change-Id: Ide4ddbff14d3745c6f2c2f9b14b00da790a6cec6
Reviewed-on: http://gerrit.cloudera.org:8080/10352
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3855328b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3855328b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3855328b
Branch: refs/heads/master
Commit: 3855328b73c28d9ce4dbe5e1773bd6abde2e935a
Parents: 07d6b5f
Author: Dan Burkert <da...@apache.org>
Authored: Tue May 8 17:00:22 2018 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Thu May 10 22:53:24 2018 +0000
----------------------------------------------------------------------
thirdparty/build-definitions.sh | 29 ++++++++++++++++-------------
1 file changed, 16 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/3855328b/thirdparty/build-definitions.sh
----------------------------------------------------------------------
diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh
index fe3f477..ebb4f72 100644
--- a/thirdparty/build-definitions.sh
+++ b/thirdparty/build-definitions.sh
@@ -252,6 +252,11 @@ build_llvm() {
TOOLS_ARGS="$TOOLS_ARGS -D${arg}=OFF"
done
+ # Remove '-nostdinc++' from the cflags for clang, since it already
+ # handles this when passing --stdlib=libc++ and passing this confuses
+ # the check for -fPIC.
+ CLANG_CXXFLAGS=$(echo "$EXTRA_CXXFLAGS" | sed -e 's,-nostdinc++,,g;')
+
case $BUILD_TYPE in
"normal")
# Default build: core LLVM libraries, clang, compiler-rt, and all tools.
@@ -269,6 +274,15 @@ build_llvm() {
if [ -n "$GCC_INSTALL_PREFIX" ]; then
TOOLS_ARGS="$TOOLS_ARGS -DGCC_INSTALL_PREFIX=$GCC_INSTALL_PREFIX"
fi
+
+ # Depend on zlib from the thirdparty tree. It's an optional dependency for
+ # LLVM, but a required [1] one for IWYU. When TSAN is enabled these flags
+ # are already set by build-thirdparty.sh in order to support the
+ # thirdparty libc++, so it's not necessary to set them again.
+ #
+ # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539
+ CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include"
+ CLANG_LDFLAGS="$CLANG_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib"
;;
"tsan")
# Build just the core LLVM libraries, dependent on libc++.
@@ -303,18 +317,6 @@ build_llvm() {
$PREFIX/lib/clang/ \
$PREFIX/lib/cmake/{llvm,clang}
- # Remove '-nostdinc++' from the cflags for clang, since it already
- # handles this when passing --stdlib=libc++ and passing this confuses
- # the check for -fPIC.
- CLANG_CXXFLAGS=$(echo "$EXTRA_CXXFLAGS" | sed -e 's,-nostdinc++,,g;')
-
- # Depend on zlib from the thirdparty tree. It's an optional dependency for
- # LLVM, but a required [1] one for IWYU.
- #
- # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539
- CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include"
- CLANG_LDFLAGS="$EXTRA_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib"
-
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
@@ -324,7 +326,8 @@ build_llvm() {
-DLLVM_INCLUDE_UTILS=OFF \
-DLLVM_TARGETS_TO_BUILD=X86 \
-DLLVM_ENABLE_RTTI=ON \
- -DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS $CLANG_LDFLAGS" \
+ -DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS $EXTRA_LDFLAGS" \
+ -DCMAKE_EXE_LINKER_FLAGS="$CLANG_LDFLAGS" \
-DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
$TOOLS_ARGS \
$EXTRA_CMAKE_FLAGS \