You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2017/10/26 16:12:35 UTC
[1/4] kudu git commit: Add missing documentation for reactor latency
metric
Repository: kudu
Updated Branches:
refs/heads/master e63d2be62 -> c8603ab77
Add missing documentation for reactor latency metric
Change-Id: I93f18b3f820e2d72648a7142ef2fb344d7db289f
Reviewed-on: http://gerrit.cloudera.org:8080/8383
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/69b82b5c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/69b82b5c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/69b82b5c
Branch: refs/heads/master
Commit: 69b82b5cc7548519f4474af1d0436ac5ffb8c943
Parents: e63d2be
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Oct 25 12:48:51 2017 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Oct 25 20:28:46 2017 +0000
----------------------------------------------------------------------
src/kudu/rpc/reactor.cc | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/69b82b5c/src/kudu/rpc/reactor.cc
----------------------------------------------------------------------
diff --git a/src/kudu/rpc/reactor.cc b/src/kudu/rpc/reactor.cc
index c9cd92a..617d23d 100644
--- a/src/kudu/rpc/reactor.cc
+++ b/src/kudu/rpc/reactor.cc
@@ -96,7 +96,11 @@ METRIC_DEFINE_histogram(server, reactor_load_percent,
METRIC_DEFINE_histogram(server, reactor_active_latency_us,
"Reactor Thread Active Latency",
kudu::MetricUnit::kMicroseconds,
- "TODO", 1000000, 2);
+ "Histogram of the wall clock time for reactor thread wake-ups. "
+ "The reactor thread is responsible for all network I/O and "
+ "therefore outliers in this latency histogram directly contribute "
+ "to the latency of both inbound and outbound RPCs.",
+ 1000000, 2);
namespace kudu {
namespace rpc {
[3/4] kudu git commit: thirdparty: bump crcutil hash
Posted by ad...@apache.org.
thirdparty: bump crcutil hash
The new revision includes a patch that detects gcc correctly in more
circumstances. In my experience the detection (and build) would fail when
thirdparty was rebuilt by make directly, or when ccache was in use. This
should address that.
I also published the internal repo externally, something we should have done
a long time ago.
Change-Id: I60c2ad24ee6919ec1a1c7948c0ce5df468adde6b
Reviewed-on: http://gerrit.cloudera.org:8080/8390
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d2be2834
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d2be2834
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d2be2834
Branch: refs/heads/master
Commit: d2be2834abccb831df63c3f978f32becbed32ee9
Parents: e03b8e4
Author: Adar Dembo <ad...@cloudera.com>
Authored: Wed Oct 25 15:41:29 2017 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Thu Oct 26 16:11:56 2017 +0000
----------------------------------------------------------------------
thirdparty/vars.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/d2be2834/thirdparty/vars.sh
----------------------------------------------------------------------
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 4451e8a..7fe0e0e 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -124,13 +124,13 @@ CURL_NAME=curl-$CURL_VERSION
CURL_SOURCE=$TP_SOURCE_DIR/$CURL_NAME
# Hash of the crcutil git revision to use.
-# (from http://github.mtv.cloudera.com/CDH/crcutil)
+# (from http://github.com/adembo/crcutil)
#
# To re-build this tarball use the following in the crcutil repo:
# export NAME=crcutil-$(git rev-parse HEAD)
# git archive HEAD --prefix=$NAME/ -o /tmp/$NAME.tar.gz
# s3cmd put -P /tmp/$NAME.tar.gz s3://cloudera-thirdparty-libs/$NAME.tar.gz
-CRCUTIL_VERSION=440ba7babeff77ffad992df3a10c767f184e946e
+CRCUTIL_VERSION=42148a6df6986a257ab21c80f8eca2e54544ac4d
CRCUTIL_NAME=crcutil-$CRCUTIL_VERSION
CRCUTIL_SOURCE=$TP_SOURCE_DIR/$CRCUTIL_NAME
[4/4] kudu git commit: docs: update security doc on --redact flag
usage
Posted by ad...@apache.org.
docs: update security doc on --redact flag usage
Change-Id: I5356aba4277fbea957289db2790ea1353bab4ba3
Reviewed-on: http://gerrit.cloudera.org:8080/8394
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c8603ab7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c8603ab7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c8603ab7
Branch: refs/heads/master
Commit: c8603ab77bfc9906dcf1affdd7b0b1c77d21537b
Parents: d2be283
Author: hahao <ha...@cloudera.com>
Authored: Wed Oct 25 16:53:32 2017 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Thu Oct 26 16:12:11 2017 +0000
----------------------------------------------------------------------
docs/security.adoc | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/c8603ab7/docs/security.adoc
----------------------------------------------------------------------
diff --git a/docs/security.adoc b/docs/security.adoc
index 85c3529..449106d 100644
--- a/docs/security.adoc
+++ b/docs/security.adoc
@@ -177,8 +177,9 @@ WARNING: Disabling the web UI will also disable REST endpoints such as
== Log Security
To prevent sensitive data from being included in Kudu server logs, all row data
-is redacted by default. This feature can be turned off configuring the
-`--redact` flag.
+is redacted by default. By setting the `--redact=log` flag, redaction will be
+disabled in the web UI but retained for server logs. Alternatively, `--redact=none`
+can be used to disable redaction completely.
// TODO(dan): add link to configuration reference.
[[configuration]]
[2/4] kudu git commit: data_dirs: be permissive if RefreshIsFull fails
Posted by ad...@apache.org.
data_dirs: be permissive if RefreshIsFull fails
In testing disk failures, at the DataDirManager-level, RefreshIsFull()
is a point of failure that can fail various functions, e.g. for block
placement. This doesn't need to be the case; it should be sufficient to
instead just not include the failed directory candidate.
This is favorable to returning early because RefreshIsFull() may cause
these functions to return before any error-handling has happened, and
any disk-failure handling will be triggered regardless the next time a
block is written to the bad disk.
Change-Id: Iffd0342e52e9f6a76ba17c179a36a8a971b94786
Reviewed-on: http://gerrit.cloudera.org:8080/8389
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e03b8e44
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e03b8e44
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e03b8e44
Branch: refs/heads/master
Commit: e03b8e4437920239ca5542143d36c7bd5a496eb2
Parents: 69b82b5
Author: Andrew Wong <aw...@cloudera.com>
Authored: Wed Oct 25 14:04:29 2017 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Thu Oct 26 16:11:42 2017 +0000
----------------------------------------------------------------------
src/kudu/fs/data_dirs.cc | 27 ++++++++++++++-------------
src/kudu/fs/data_dirs.h | 2 +-
2 files changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/e03b8e44/src/kudu/fs/data_dirs.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index 5e9b0d0..1ae0e9f 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -670,7 +670,7 @@ Status DataDirManager::CreateDataDirGroup(const string& tablet_id,
return Status::IOError("No healthy data directories available", "", ENODEV);
}
}
- RETURN_NOT_OK(GetDirsForGroupUnlocked(group_target_size, &group_indices));
+ GetDirsForGroupUnlocked(group_target_size, &group_indices);
if (PREDICT_FALSE(group_indices.empty())) {
return Status::IOError("All healthy data directories are full", "", ENOSPC);
}
@@ -724,8 +724,9 @@ Status DataDirManager::GetNextDataDir(const CreateBlockOptions& opts, DataDir**
for (int i : random_indices) {
int uuid_idx = (*group_uuid_indices)[i];
DataDir* candidate = FindOrDie(data_dir_by_uuid_idx_, uuid_idx);
- RETURN_NOT_OK(candidate->RefreshIsFull(DataDir::RefreshMode::EXPIRED_ONLY));
- if (!candidate->is_full()) {
+ Status s = candidate->RefreshIsFull(DataDir::RefreshMode::EXPIRED_ONLY);
+ WARN_NOT_OK(s, Substitute("failed to refresh fullness of $0", candidate->dir()));
+ if (s.ok() && !candidate->is_full()) {
*dir = candidate;
return Status::OK();
}
@@ -740,8 +741,8 @@ Status DataDirManager::GetNextDataDir(const CreateBlockOptions& opts, DataDir**
metrics_->data_dirs_full->value(), dirs_state_str);
}
return Status::IOError(Substitute("No directories available to add to $0directory group ($1 "
- "dirs total, $2).", tablet_id_str, data_dirs_.size(), dirs_state_str),
- "", ENOSPC);
+ "dirs total, $2).", tablet_id_str, data_dirs_.size(), dirs_state_str),
+ "", ENOSPC);
}
void DataDirManager::DeleteDataDirGroup(const std::string& tablet_id) {
@@ -768,19 +769,20 @@ bool DataDirManager::GetDataDirGroupPB(const std::string& tablet_id,
return false;
}
-Status DataDirManager::GetDirsForGroupUnlocked(int target_size,
- vector<int>* group_indices) {
+void DataDirManager::GetDirsForGroupUnlocked(int target_size,
+ vector<int>* group_indices) {
DCHECK(dir_group_lock_.is_locked());
vector<int> candidate_indices;
for (auto& e : data_dir_by_uuid_idx_) {
if (ContainsKey(failed_data_dirs_, e.first)) {
continue;
}
- RETURN_NOT_OK(e.second->RefreshIsFull(DataDir::RefreshMode::ALWAYS));
- // TODO(awong): If a disk is unhealthy at the time of group creation, the
- // resulting group may be below targeted size. Add functionality to resize
- // groups. See KUDU-2040 for more details.
- if (!e.second->is_full()) {
+ Status s = e.second->RefreshIsFull(DataDir::RefreshMode::ALWAYS);
+ WARN_NOT_OK(s, Substitute("failed to refresh fullness of $0", e.second->dir()));
+ if (s.ok() && !e.second->is_full()) {
+ // TODO(awong): If a disk is unhealthy at the time of group creation, the
+ // resulting group may be below targeted size. Add functionality to
+ // resize groups. See KUDU-2040 for more details.
candidate_indices.push_back(e.first);
}
}
@@ -796,7 +798,6 @@ Status DataDirManager::GetDirsForGroupUnlocked(int target_size,
candidate_indices.erase(candidate_indices.begin() + 1);
}
}
- return Status::OK();
}
DataDir* DataDirManager::FindDataDirByUuidIndex(int uuid_idx) const {
http://git-wip-us.apache.org/repos/asf/kudu/blob/e03b8e44/src/kudu/fs/data_dirs.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs.h b/src/kudu/fs/data_dirs.h
index 1dca128..0ed4389 100644
--- a/src/kudu/fs/data_dirs.h
+++ b/src/kudu/fs/data_dirs.h
@@ -414,7 +414,7 @@ class DataDirManager {
// added. Although this function does not itself change DataDirManager state,
// its expected usage warrants that it is called within the scope of a
// lock_guard of dir_group_lock_.
- Status GetDirsForGroupUnlocked(int target_size, std::vector<int>* group_indices);
+ void GetDirsForGroupUnlocked(int target_size, std::vector<int>* group_indices);
// Goes through the data dirs in 'uuid_indices' and populates
// 'healthy_indices' with those that haven't failed.