You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by wd...@apache.org on 2019/02/15 18:26:47 UTC
[kudu] branch master updated (c23dc84 -> 794a9ec)
This is an automated email from the ASF dual-hosted git repository.
wdberkeley pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from c23dc84 Fix compilation warnings under debian8.9
new 3e5fb43 [tools] Add tool to list data dirs for a tablet
new bd5926e [tools] Add tablet data dirs to 'kudu remote replica list'
new 794a9ec docs: fix two broken links in the installation doc
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/installation.adoc | 4 +-
src/kudu/fs/data_dirs.cc | 25 ++++++++
src/kudu/fs/data_dirs.h | 5 ++
src/kudu/tablet/tablet.proto | 1 +
src/kudu/tablet/tablet_replica.cc | 15 ++++-
src/kudu/tools/kudu-tool-test.cc | 89 ++++++++++++++++++++++++++++
src/kudu/tools/tool_action_local_replica.cc | 29 ++++++++-
src/kudu/tools/tool_action_remote_replica.cc | 6 ++
8 files changed, 170 insertions(+), 4 deletions(-)
[kudu] 02/03: [tools] Add tablet data dirs to 'kudu remote replica
list'
Posted by wd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit bd5926ee50cbde7b86cbd2342b61e8370836fd4a
Author: Will Berkeley <wd...@gmail.org>
AuthorDate: Mon Feb 11 12:50:51 2019 -0800
[tools] Add tablet data dirs to 'kudu remote replica list'
This is useful to find out the data dirs for a tablet on a remote server
while the server is running. For example, it could be used to
investigate performance issues related to the devices on which the data
for the tablet is stored.
Change-Id: I7e509b5c080a76a4110a32ecc073418c28aa0633
Reviewed-on: http://gerrit.cloudera.org:8080/12449
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Will Berkeley <wd...@gmail.com>
---
src/kudu/tablet/tablet.proto | 1 +
src/kudu/tablet/tablet_replica.cc | 15 ++++++++-
src/kudu/tools/kudu-tool-test.cc | 47 ++++++++++++++++++++++++++++
src/kudu/tools/tool_action_remote_replica.cc | 6 ++++
4 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/src/kudu/tablet/tablet.proto b/src/kudu/tablet/tablet.proto
index 9706c9c..7d347e1 100644
--- a/src/kudu/tablet/tablet.proto
+++ b/src/kudu/tablet/tablet.proto
@@ -101,4 +101,5 @@ message TabletStatusPB {
optional bytes end_key = 6;
optional PartitionPB partition = 9;
optional int64 estimated_on_disk_size = 7;
+ repeated bytes data_dirs = 10;
}
diff --git a/src/kudu/tablet/tablet_replica.cc b/src/kudu/tablet/tablet_replica.cc
index 4175f5b..c7b231b 100644
--- a/src/kudu/tablet/tablet_replica.cc
+++ b/src/kudu/tablet/tablet_replica.cc
@@ -38,6 +38,7 @@
#include "kudu/consensus/log_anchor_registry.h"
#include "kudu/consensus/opid.pb.h"
#include "kudu/consensus/raft_consensus.h"
+#include "kudu/gutil/basictypes.h"
#include "kudu/gutil/bind.h"
#include "kudu/gutil/bind_helpers.h"
#include "kudu/gutil/port.h"
@@ -436,11 +437,23 @@ void TabletReplica::GetTabletStatusPB(TabletStatusPB* status_pb_out) const {
status_pb_out->set_state(state_);
status_pb_out->set_last_status(last_status_);
}
- status_pb_out->set_tablet_id(meta_->tablet_id());
+ const string& tablet_id = meta_->tablet_id();
+ status_pb_out->set_tablet_id(tablet_id);
status_pb_out->set_table_name(meta_->table_name());
meta_->partition().ToPB(status_pb_out->mutable_partition());
status_pb_out->set_tablet_data_state(meta_->tablet_data_state());
status_pb_out->set_estimated_on_disk_size(OnDiskSize());
+ // There are circumstances where the call to 'FindDataDirsByTabletId' may
+ // fail, like if the tablet is tombstoned or failed. It's alright to return
+ // an empty 'data_dirs' in this case-- the state and last status will inform
+ // the caller.
+ vector<string> data_dirs;
+ ignore_result(
+ meta_->fs_manager()->dd_manager()->FindDataDirsByTabletId(tablet_id,
+ &data_dirs));
+ for (auto& dir : data_dirs) {
+ status_pb_out->add_data_dirs(std::move(dir));
+ }
}
Status TabletReplica::RunLogGC() {
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index e0a3a91..f6542c0 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -1893,6 +1893,53 @@ TEST_F(ToolTest, TestRemoteReplicaCopy) {
tablet::RUNNING, kTimeout));
}
+// Test the 'kudu remote_replica list' tool.
+TEST_F(ToolTest, TestRemoteReplicaList) {
+ MonoDelta kTimeout = MonoDelta::FromSeconds(30);
+ const int kNumTservers = 1;
+ const int kNumTablets = 1;
+ ExternalMiniClusterOptions opts;
+ opts.num_data_dirs = 3;
+ opts.num_tablet_servers = kNumTservers;
+ NO_FATALS(StartExternalMiniCluster(std::move(opts)));
+
+ // TestWorkLoad.Setup() internally generates a table.
+ TestWorkload workload(cluster_.get());
+ workload.set_num_tablets(kNumTablets);
+ workload.set_num_replicas(kNumTservers);
+ workload.Setup();
+
+ TServerDetails* ts = ts_map_[cluster_->tablet_server(0)->uuid()];
+ vector<ListTabletsResponsePB::StatusAndSchemaPB> tablets;
+ ASSERT_OK(WaitForNumTabletsOnTS(ts, kNumTablets, kTimeout, &tablets));
+ const string& ts_addr = cluster_->tablet_server(0)->bound_rpc_addr().ToString();
+ string stdout;
+ NO_FATALS(RunActionStdoutString(
+ Substitute("remote_replica list $0", ts_addr), &stdout));
+ const auto& tablet_status = tablets[0].tablet_status();
+
+ // Some fields like state or estimated on disk size may vary. Just check a
+ // few whose values we should know exactly.
+ ASSERT_STR_CONTAINS(stdout,
+ Substitute("Tablet id: $0", tablet_status.tablet_id()));
+ ASSERT_STR_CONTAINS(stdout,
+ Substitute("Table name: $0", workload.table_name()));
+ ASSERT_STR_CONTAINS(stdout,
+ Substitute("Data dirs: $0", JoinStrings(tablet_status.data_dirs(), ", ")));
+
+ // Tombstone the replica and try again.
+ ASSERT_OK(DeleteTablet(ts, tablet_status.tablet_id(),
+ TabletDataState::TABLET_DATA_TOMBSTONED, kTimeout));
+ NO_FATALS(RunActionStdoutString(
+ Substitute("remote_replica list $0", ts_addr), &stdout));
+ ASSERT_STR_CONTAINS(stdout,
+ Substitute("Tablet id: $0", tablet_status.tablet_id()));
+ ASSERT_STR_CONTAINS(stdout,
+ Substitute("Table name: $0", workload.table_name()));
+ ASSERT_STR_CONTAINS(stdout,
+ Substitute("Data dirs: $0", JoinStrings(tablet_status.data_dirs(), ", ")));
+}
+
// Test 'kudu local_replica delete' tool with --clean_unsafe flag for
// deleting the tablet from the tablet server.
TEST_F(ToolTest, TestLocalReplicaDelete) {
diff --git a/src/kudu/tools/tool_action_remote_replica.cc b/src/kudu/tools/tool_action_remote_replica.cc
index 9ffb195..6323f9b 100644
--- a/src/kudu/tools/tool_action_remote_replica.cc
+++ b/src/kudu/tools/tool_action_remote_replica.cc
@@ -43,6 +43,7 @@
#include "kudu/consensus/metadata.pb.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/strings/human_readable.h"
+#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/rpc/rpc_controller.h"
#include "kudu/server/server_base.pb.h"
@@ -305,6 +306,11 @@ Status ListReplicas(const RunnerContext& context) {
cout << "Estimated on disk size: "
<< HumanReadableNumBytes::ToString(rs.estimated_on_disk_size()) << endl;
}
+ if (rs.data_dirs_size() != 0) {
+ cout << "Data dirs: " << JoinStrings(rs.data_dirs(), ", ") << endl;
+ } else {
+ cout << "Data dirs: <not available>" << endl;
+ }
cout << "Schema: " << schema.ToString() << endl;
}
[kudu] 01/03: [tools] Add tool to list data dirs for a tablet
Posted by wd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 3e5fb43cd4349cfc019b717bce5876c802f97683
Author: Will Berkeley <wd...@gmail.org>
AuthorDate: Fri Feb 8 14:36:35 2019 -0800
[tools] Add tool to list data dirs for a tablet
It's hard to find out what the data dirs are for a tablet, given its
tablet id, but it might be useful to do so. For example, if a user is
trying to correlate disk utilization to busy tablets, it's useful to
know which tablets will cause disk activity on which devices.
Change-Id: I6c6de565e4061eb8248b453d2230847e064040ff
Reviewed-on: http://gerrit.cloudera.org:8080/12417
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Will Berkeley <wd...@gmail.com>
---
src/kudu/fs/data_dirs.cc | 25 +++++++++++++++++
src/kudu/fs/data_dirs.h | 5 ++++
src/kudu/tools/kudu-tool-test.cc | 42 +++++++++++++++++++++++++++++
src/kudu/tools/tool_action_local_replica.cc | 29 +++++++++++++++++++-
4 files changed, 100 insertions(+), 1 deletion(-)
diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index af0cbec..4363725 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -1093,6 +1093,31 @@ set<string> DataDirManager::FindTabletsByDataDirUuidIdx(int uuid_idx) const {
return {};
}
+Status DataDirManager::FindDataDirsByTabletId(const string& tablet_id,
+ vector<string>* data_dirs) const {
+ CHECK(data_dirs);
+ DataDirGroupPB group;
+ RETURN_NOT_OK(GetDataDirGroupPB(tablet_id, &group));
+ vector<string> data_dirs_tmp;
+ data_dirs_tmp.reserve(group.uuids_size());
+ for (const auto& uuid : group.uuids()) {
+ int uuid_idx;
+ if (!FindUuidIndexByUuid(uuid, &uuid_idx)) {
+ return Status::NotFound("unable to find index for UUID", uuid);
+ }
+ const auto* data_dir = FindDataDirByUuidIndex(uuid_idx);
+ if (!data_dir) {
+ return Status::NotFound(
+ Substitute("unable to find data dir for UUID $0 with index $1",
+ uuid, uuid_idx));
+ }
+ data_dirs_tmp.emplace_back(data_dir->dir());
+ }
+ std::sort(data_dirs_tmp.begin(), data_dirs_tmp.end());
+ *data_dirs = std::move(data_dirs_tmp);
+ return Status::OK();
+}
+
void DataDirManager::MarkDataDirFailedByUuid(const string& uuid) {
int uuid_idx;
CHECK(FindUuidIndexByUuid(uuid, &uuid_idx));
diff --git a/src/kudu/fs/data_dirs.h b/src/kudu/fs/data_dirs.h
index 2b2fc75..9bc3911 100644
--- a/src/kudu/fs/data_dirs.h
+++ b/src/kudu/fs/data_dirs.h
@@ -335,6 +335,11 @@ class DataDirManager {
// returns a copy, returning an empty set if none are found.
std::set<std::string> FindTabletsByDataDirUuidIdx(int uuid_idx) const;
+ // Returns in 'data_dirs' a sorted list of the directory names for the data
+ // dirs of the tablet specified by 'tablet_id'.
+ Status FindDataDirsByTabletId(const std::string& tablet_id,
+ std::vector<std::string>* data_dirs) const;
+
// ==========================================================================
// Directory Health
// ==========================================================================
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 522c204..e0a3a91 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -552,6 +552,7 @@ TEST_F(ToolTest, TestModeHelp) {
{
const vector<string> kLocalReplicaDumpModeRegexes = {
"block_ids.*Dump the IDs of all blocks",
+ "data_dirs.*Dump the data directories",
"meta.*Dump the metadata",
"rowset.*Dump the rowset contents",
"wals.*Dump all WAL"
@@ -1174,6 +1175,47 @@ TEST_F(ToolTest, TestWalDump) {
}
}
+TEST_F(ToolTest, TestLocalReplicaDumpDataDirs) {
+ const string kTestTablet = "ffffffffffffffffffffffffffffffff";
+ const string kTestTableId = "test-table";
+ const string kTestTableName = "test-fs-data-dirs-dump-table";
+ const Schema kSchema(GetSimpleTestSchema());
+ const Schema kSchemaWithIds(SchemaBuilder(kSchema).Build());
+
+ FsManagerOpts opts;
+ opts.wal_root = GetTestPath("wal");
+ opts.data_roots = {
+ GetTestPath("data0"),
+ GetTestPath("data1"),
+ GetTestPath("data2"),
+ };
+ FsManager fs(env_, opts);
+ ASSERT_OK(fs.CreateInitialFileSystemLayout());
+ ASSERT_OK(fs.Open());
+
+ pair<PartitionSchema, Partition> partition = tablet::CreateDefaultPartition(
+ kSchemaWithIds);
+ scoped_refptr<TabletMetadata> meta;
+ ASSERT_OK(TabletMetadata::CreateNew(
+ &fs, kTestTablet, kTestTableName, kTestTableId,
+ kSchemaWithIds, partition.first, partition.second,
+ tablet::TABLET_DATA_READY,
+ /*tombstone_last_logged_opid=*/ boost::none,
+ &meta));
+ string stdout;
+ NO_FATALS(RunActionStdoutString(Substitute("local_replica dump data_dirs $0 "
+ "--fs_wal_dir=$1 "
+ "--fs_data_dirs=$2",
+ kTestTablet, opts.wal_root,
+ JoinStrings(opts.data_roots, ",")),
+ &stdout));
+ vector<string> expected;
+ for (const auto& data_root : opts.data_roots) {
+ expected.emplace_back(JoinPathSegments(data_root, "data"));
+ }
+ ASSERT_EQ(JoinStrings(expected, "\n"), stdout);
+}
+
TEST_F(ToolTest, TestLocalReplicaDumpMeta) {
const string kTestDir = GetTestPath("test");
const string kTestTablet = "ffffffffffffffffffffffffffffffff";
diff --git a/src/kudu/tools/tool_action_local_replica.cc b/src/kudu/tools/tool_action_local_replica.cc
index 21edb5e..2173acb 100644
--- a/src/kudu/tools/tool_action_local_replica.cc
+++ b/src/kudu/tools/tool_action_local_replica.cc
@@ -46,6 +46,7 @@
#include "kudu/consensus/opid.pb.h"
#include "kudu/fs/block_id.h"
#include "kudu/fs/block_manager.h"
+#include "kudu/fs/data_dirs.h"
#include "kudu/fs/fs_manager.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/ref_counted.h"
@@ -681,7 +682,23 @@ Status DumpMeta(const RunnerContext& context) {
unique_ptr<FsManager> fs_manager;
RETURN_NOT_OK(FsInit(&fs_manager));
const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
- RETURN_NOT_OK(DumpTabletMeta(fs_manager.get(), tablet_id, 0));
+ return DumpTabletMeta(fs_manager.get(), tablet_id, 0);
+}
+
+Status DumpDataDirs(const RunnerContext& context) {
+ unique_ptr<FsManager> fs_manager;
+ RETURN_NOT_OK(FsInit(&fs_manager));
+ const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
+ // Load the tablet meta to make sure the tablet's data directories are loaded
+ // into the manager.
+ scoped_refptr<TabletMetadata> unused_meta;
+ RETURN_NOT_OK(TabletMetadata::Load(fs_manager.get(), tablet_id, &unused_meta));
+ vector<string> data_dirs;
+ RETURN_NOT_OK(fs_manager->dd_manager()->FindDataDirsByTabletId(tablet_id,
+ &data_dirs));
+ for (const auto& dir : data_dirs) {
+ cout << dir << endl;
+ }
return Status::OK();
}
@@ -695,6 +712,15 @@ unique_ptr<Mode> BuildDumpMode() {
.AddOptionalParameter("fs_wal_dir")
.Build();
+ unique_ptr<Action> dump_data_dirs =
+ ActionBuilder("data_dirs", &DumpDataDirs)
+ .Description("Dump the data directories where the replica's data is stored")
+ .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
+ .AddOptionalParameter("fs_data_dirs")
+ .AddOptionalParameter("fs_metadata_dir")
+ .AddOptionalParameter("fs_wal_dir")
+ .Build();
+
unique_ptr<Action> dump_meta =
ActionBuilder("meta", &DumpMeta)
.Description("Dump the metadata of a local replica")
@@ -733,6 +759,7 @@ unique_ptr<Mode> BuildDumpMode() {
return ModeBuilder("dump")
.Description("Dump a Kudu filesystem")
.AddAction(std::move(dump_block_ids))
+ .AddAction(std::move(dump_data_dirs))
.AddAction(std::move(dump_meta))
.AddAction(std::move(dump_rowset))
.AddAction(std::move(dump_wals))
[kudu] 03/03: docs: fix two broken links in the installation doc
Posted by wd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 794a9ecf8a23652ab8d5f65b4536f3850de1d445
Author: Greg Solovyev <fi...@gmail.com>
AuthorDate: Wed Feb 13 18:20:33 2019 -0800
docs: fix two broken links in the installation doc
Change-Id: I249df2ac3a88bda9825419a852a752423a5cb04f
Reviewed-on: http://gerrit.cloudera.org:8080/12480
Tested-by: Kudu Jenkins
Reviewed-by: Grant Henke <gr...@apache.org>
---
docs/installation.adoc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/installation.adoc b/docs/installation.adoc
index 443bedb..7d219f2 100644
--- a/docs/installation.adoc
+++ b/docs/installation.adoc
@@ -34,7 +34,7 @@ from source. To run Kudu without installing anything, use the link:quickstart.ht
NOTE: Cloudera Manager users can install and manage Kudu with
link:http://www.cloudera.com/content/www/en-us/products/cloudera-manager.html[Cloudera Manager],
instead of following these instructions. For more information, see Cloudera's
-link:http://www.cloudera.com/documentation/kudu/latest/topics/kudu_installation.html[Kudu documentation].
+link:http://www.cloudera.com/documentation/enterprise/latest/topics/kudu_install_upgrade_intro.html[Kudu documentation].
.Upgrading Kudu
To upgrade Kudu from a previous version, see <<upgrade>>.
@@ -573,7 +573,7 @@ $ sudo xcode-select --install
.macOS Known Issues
====
Kudu support for macOS is experimental, and should only be used for development.
-See link:https://issues.apache.org/browse/KUDU-1219[macOS Limitations & Known Issues]
+See link:https://issues.apache.org/jira/browse/KUDU-1219[macOS Limitations & Known Issues]
for more information.
====