You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2017/06/23 23:48:42 UTC
[2/2] kudu git commit: KUDU-2052: disable hole repunching at startup
on xfs filesystems
KUDU-2052: disable hole repunching at startup on xfs filesystems
As a quick and dirty workaround for (what we believe to be) the root cause
of KUDU-2052, let's disable hole repunching at startup for containers that
reside on xfs filesystems. This toggle is gated behind a gflag which is
disabled (i.e. hole repunching is enabled) in the CLI and unit tests.
Once the real fix for KUDU-2052 (i.e. use the faster ioctl-based hole punch
method) is merged, this will be reverted.
To test, I ran ctest on a loopback mounted xfs filesystem. I also created a
single node deployment on this filesystem, loaded ~10k (live and dead)
blocks into it across ~100 containers. Then I restarted the tserver under
strace, with metadata compaction disabled and with
--log-container-max-blocks set to a special value so that the LBM would
consider all of the containers full. The strace log showed that all of the
container data files were all truncated at startup (because they were full
and had extra space at the end), but there were no hole punches.
Change-Id: Ifc0e935775da0817452ee90f98eea4edea3259d4
Reviewed-on: http://gerrit.cloudera.org:8080/7279
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/410763af
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/410763af
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/410763af
Branch: refs/heads/master
Commit: 410763afa61252fa8c7125f66429bad602db795a
Parents: 9fdf77a
Author: Adar Dembo <ad...@cloudera.com>
Authored: Fri Jun 23 11:32:30 2017 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Jun 23 23:48:13 2017 +0000
----------------------------------------------------------------------
src/kudu/fs/log_block_manager-test.cc | 4 ++++
src/kudu/fs/log_block_manager.cc | 14 +++++++++++++-
src/kudu/tools/tool_action_fs.cc | 6 ++++++
3 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/410763af/src/kudu/fs/log_block_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager-test.cc b/src/kudu/fs/log_block_manager-test.cc
index f3f1cfe..334db10 100644
--- a/src/kudu/fs/log_block_manager-test.cc
+++ b/src/kudu/fs/log_block_manager-test.cc
@@ -46,6 +46,7 @@ using strings::Substitute;
DECLARE_int64(block_manager_max_open_files);
DECLARE_bool(cache_force_single_shard);
+DECLARE_bool(log_block_manager_disable_hole_repunching_on_xfs);
DECLARE_double(log_container_excess_space_before_cleanup_fraction);
DECLARE_double(log_container_live_metadata_before_compact_ratio);
DECLARE_int64(log_container_max_blocks);
@@ -938,6 +939,9 @@ TEST_F(LogBlockManagerTest, TestRepairUnpunchedBlocks) {
// preallocation buffer.
FLAGS_log_container_preallocate_bytes = 0;
+ // Disable the KUDU-2052 workaround for the test.
+ FLAGS_log_block_manager_disable_hole_repunching_on_xfs = false;
+
// Create one container.
unique_ptr<WritableBlock> block;
ASSERT_OK(bm_->CreateBlock(test_block_opts_, &block));
http://git-wip-us.apache.org/repos/asf/kudu/blob/410763af/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index bc2ae7c..516f5e0 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -96,6 +96,13 @@ DEFINE_bool(log_block_manager_test_hole_punching, true,
TAG_FLAG(log_block_manager_test_hole_punching, advanced);
TAG_FLAG(log_block_manager_test_hole_punching, unsafe);
+DEFINE_bool(log_block_manager_disable_hole_repunching_on_xfs, true,
+ "Whether to disable hole repunching at startup of full containers "
+ "on XFS filesystems that consume excess space. This is a "
+ "workaround for KUDU-2052.");
+TAG_FLAG(log_block_manager_disable_hole_repunching_on_xfs, advanced);
+TAG_FLAG(log_block_manager_disable_hole_repunching_on_xfs, unsafe);
+
METRIC_DEFINE_gauge_uint64(server, log_block_manager_bytes_under_management,
"Bytes Under Management",
kudu::MetricUnit::kBytes,
@@ -1998,7 +2005,12 @@ void LogBlockManager::OpenDataDir(DataDir* dir,
// If the container is to be deleted outright, don't bother repunching
// its blocks. The report entry remains, however, so it's clear that
// there was a space discrepancy.
- if (container->live_blocks()) {
+ //
+ // KUDU-2052: hole punching is generally skipped on XFS due to
+ // pathologically slow fallocate() behavior.
+ if (container->live_blocks() &&
+ (dir->fs_type() != DataDirFsType::XFS ||
+ !FLAGS_log_block_manager_disable_hole_repunching_on_xfs)) {
need_repunching.insert(need_repunching.end(),
dead_blocks.begin(), dead_blocks.end());
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/410763af/src/kudu/tools/tool_action_fs.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_fs.cc b/src/kudu/tools/tool_action_fs.cc
index fc4ef59..615e88c 100644
--- a/src/kudu/tools/tool_action_fs.cc
+++ b/src/kudu/tools/tool_action_fs.cc
@@ -72,6 +72,12 @@ using tablet::TabletMetadata;
namespace {
Status Check(const RunnerContext& /*context*/) {
+ // The symptom of KUDU-2052 is pathological slowness; we can tolerate that
+ // when checking the filesystem from the CLI.
+ CHECK_NE("", google::SetCommandLineOptionWithMode(
+ "log_block_manager_disable_hole_repunching_on_xfs", "false",
+ google::FlagSettingMode::SET_FLAGS_DEFAULT));
+
FsManagerOpts opts;
opts.read_only = !FLAGS_repair;
FsManager fs_manager(Env::Default(), opts);