You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by wd...@apache.org on 2019/02/15 18:26:47 UTC

[kudu] branch master updated (c23dc84 -> 794a9ec)

This is an automated email from the ASF dual-hosted git repository.

wdberkeley pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.


    from c23dc84  Fix compilation warnings under debian8.9
     new 3e5fb43  [tools] Add tool to list data dirs for a tablet
     new bd5926e  [tools] Add tablet data dirs to 'kudu remote replica list'
     new 794a9ec  docs: fix two broken links in the installation doc

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 docs/installation.adoc                       |  4 +-
 src/kudu/fs/data_dirs.cc                     | 25 ++++++++
 src/kudu/fs/data_dirs.h                      |  5 ++
 src/kudu/tablet/tablet.proto                 |  1 +
 src/kudu/tablet/tablet_replica.cc            | 15 ++++-
 src/kudu/tools/kudu-tool-test.cc             | 89 ++++++++++++++++++++++++++++
 src/kudu/tools/tool_action_local_replica.cc  | 29 ++++++++-
 src/kudu/tools/tool_action_remote_replica.cc |  6 ++
 8 files changed, 170 insertions(+), 4 deletions(-)


[kudu] 02/03: [tools] Add tablet data dirs to 'kudu remote replica list'

Posted by wd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit bd5926ee50cbde7b86cbd2342b61e8370836fd4a
Author: Will Berkeley <wd...@gmail.org>
AuthorDate: Mon Feb 11 12:50:51 2019 -0800

    [tools] Add tablet data dirs to 'kudu remote replica list'
    
    This is useful to find out the data dirs for a tablet on a remote server
    while the server is running. For example, it could be used to
    investigate performance issues related to the devices on which the data
    for the tablet is stored.
    
    Change-Id: I7e509b5c080a76a4110a32ecc073418c28aa0633
    Reviewed-on: http://gerrit.cloudera.org:8080/12449
    Reviewed-by: Adar Dembo <ad...@cloudera.com>
    Tested-by: Will Berkeley <wd...@gmail.com>
---
 src/kudu/tablet/tablet.proto                 |  1 +
 src/kudu/tablet/tablet_replica.cc            | 15 ++++++++-
 src/kudu/tools/kudu-tool-test.cc             | 47 ++++++++++++++++++++++++++++
 src/kudu/tools/tool_action_remote_replica.cc |  6 ++++
 4 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/src/kudu/tablet/tablet.proto b/src/kudu/tablet/tablet.proto
index 9706c9c..7d347e1 100644
--- a/src/kudu/tablet/tablet.proto
+++ b/src/kudu/tablet/tablet.proto
@@ -101,4 +101,5 @@ message TabletStatusPB {
   optional bytes end_key = 6;
   optional PartitionPB partition = 9;
   optional int64 estimated_on_disk_size = 7;
+  repeated bytes data_dirs = 10;
 }
diff --git a/src/kudu/tablet/tablet_replica.cc b/src/kudu/tablet/tablet_replica.cc
index 4175f5b..c7b231b 100644
--- a/src/kudu/tablet/tablet_replica.cc
+++ b/src/kudu/tablet/tablet_replica.cc
@@ -38,6 +38,7 @@
 #include "kudu/consensus/log_anchor_registry.h"
 #include "kudu/consensus/opid.pb.h"
 #include "kudu/consensus/raft_consensus.h"
+#include "kudu/gutil/basictypes.h"
 #include "kudu/gutil/bind.h"
 #include "kudu/gutil/bind_helpers.h"
 #include "kudu/gutil/port.h"
@@ -436,11 +437,23 @@ void TabletReplica::GetTabletStatusPB(TabletStatusPB* status_pb_out) const {
     status_pb_out->set_state(state_);
     status_pb_out->set_last_status(last_status_);
   }
-  status_pb_out->set_tablet_id(meta_->tablet_id());
+  const string& tablet_id = meta_->tablet_id();
+  status_pb_out->set_tablet_id(tablet_id);
   status_pb_out->set_table_name(meta_->table_name());
   meta_->partition().ToPB(status_pb_out->mutable_partition());
   status_pb_out->set_tablet_data_state(meta_->tablet_data_state());
   status_pb_out->set_estimated_on_disk_size(OnDiskSize());
+  // There are circumstances where the call to 'FindDataDirsByTabletId' may
+  // fail, like if the tablet is tombstoned or failed. It's alright to return
+  // an empty 'data_dirs' in this case-- the state and last status will inform
+  // the caller.
+  vector<string> data_dirs;
+  ignore_result(
+      meta_->fs_manager()->dd_manager()->FindDataDirsByTabletId(tablet_id,
+                                                                &data_dirs));
+  for (auto& dir : data_dirs) {
+    status_pb_out->add_data_dirs(std::move(dir));
+  }
 }
 
 Status TabletReplica::RunLogGC() {
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index e0a3a91..f6542c0 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -1893,6 +1893,53 @@ TEST_F(ToolTest, TestRemoteReplicaCopy) {
                                    tablet::RUNNING, kTimeout));
 }
 
+// Test the 'kudu remote_replica list' tool.
+TEST_F(ToolTest, TestRemoteReplicaList) {
+  MonoDelta kTimeout = MonoDelta::FromSeconds(30);
+  const int kNumTservers = 1;
+  const int kNumTablets = 1;
+  ExternalMiniClusterOptions opts;
+  opts.num_data_dirs = 3;
+  opts.num_tablet_servers = kNumTservers;
+  NO_FATALS(StartExternalMiniCluster(std::move(opts)));
+
+  // TestWorkLoad.Setup() internally generates a table.
+  TestWorkload workload(cluster_.get());
+  workload.set_num_tablets(kNumTablets);
+  workload.set_num_replicas(kNumTservers);
+  workload.Setup();
+
+  TServerDetails* ts = ts_map_[cluster_->tablet_server(0)->uuid()];
+  vector<ListTabletsResponsePB::StatusAndSchemaPB> tablets;
+  ASSERT_OK(WaitForNumTabletsOnTS(ts, kNumTablets, kTimeout, &tablets));
+  const string& ts_addr = cluster_->tablet_server(0)->bound_rpc_addr().ToString();
+  string stdout;
+  NO_FATALS(RunActionStdoutString(
+        Substitute("remote_replica list $0", ts_addr), &stdout));
+  const auto& tablet_status = tablets[0].tablet_status();
+
+  // Some fields like state or estimated on disk size may vary. Just check a
+  // few whose values we should know exactly.
+  ASSERT_STR_CONTAINS(stdout,
+                      Substitute("Tablet id: $0", tablet_status.tablet_id()));
+  ASSERT_STR_CONTAINS(stdout,
+                      Substitute("Table name: $0", workload.table_name()));
+  ASSERT_STR_CONTAINS(stdout,
+      Substitute("Data dirs: $0", JoinStrings(tablet_status.data_dirs(), ", ")));
+
+  // Tombstone the replica and try again.
+  ASSERT_OK(DeleteTablet(ts, tablet_status.tablet_id(),
+                         TabletDataState::TABLET_DATA_TOMBSTONED, kTimeout));
+  NO_FATALS(RunActionStdoutString(
+        Substitute("remote_replica list $0", ts_addr), &stdout));
+  ASSERT_STR_CONTAINS(stdout,
+                      Substitute("Tablet id: $0", tablet_status.tablet_id()));
+  ASSERT_STR_CONTAINS(stdout,
+                      Substitute("Table name: $0", workload.table_name()));
+  ASSERT_STR_CONTAINS(stdout,
+      Substitute("Data dirs: $0", JoinStrings(tablet_status.data_dirs(), ", ")));
+}
+
 // Test 'kudu local_replica delete' tool with --clean_unsafe flag for
 // deleting the tablet from the tablet server.
 TEST_F(ToolTest, TestLocalReplicaDelete) {
diff --git a/src/kudu/tools/tool_action_remote_replica.cc b/src/kudu/tools/tool_action_remote_replica.cc
index 9ffb195..6323f9b 100644
--- a/src/kudu/tools/tool_action_remote_replica.cc
+++ b/src/kudu/tools/tool_action_remote_replica.cc
@@ -43,6 +43,7 @@
 #include "kudu/consensus/metadata.pb.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/strings/human_readable.h"
+#include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/rpc/rpc_controller.h"
 #include "kudu/server/server_base.pb.h"
@@ -305,6 +306,11 @@ Status ListReplicas(const RunnerContext& context) {
       cout << "Estimated on disk size: "
            << HumanReadableNumBytes::ToString(rs.estimated_on_disk_size()) << endl;
     }
+    if (rs.data_dirs_size() != 0) {
+      cout << "Data dirs: " << JoinStrings(rs.data_dirs(), ", ") << endl;
+    } else {
+      cout << "Data dirs: <not available>" << endl;
+    }
     cout << "Schema: " << schema.ToString() << endl;
   }
 


[kudu] 01/03: [tools] Add tool to list data dirs for a tablet

Posted by wd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 3e5fb43cd4349cfc019b717bce5876c802f97683
Author: Will Berkeley <wd...@gmail.org>
AuthorDate: Fri Feb 8 14:36:35 2019 -0800

    [tools] Add tool to list data dirs for a tablet
    
    It's hard to find out what the data dirs are for a tablet, given its
    tablet id, but it might be useful to do so. For example, if a user is
    trying to correlate disk utilization to busy tablets, it's useful to
    know which tablets will cause disk activity on which devices.
    
    Change-Id: I6c6de565e4061eb8248b453d2230847e064040ff
    Reviewed-on: http://gerrit.cloudera.org:8080/12417
    Reviewed-by: Andrew Wong <aw...@cloudera.com>
    Reviewed-by: Adar Dembo <ad...@cloudera.com>
    Tested-by: Will Berkeley <wd...@gmail.com>
---
 src/kudu/fs/data_dirs.cc                    | 25 +++++++++++++++++
 src/kudu/fs/data_dirs.h                     |  5 ++++
 src/kudu/tools/kudu-tool-test.cc            | 42 +++++++++++++++++++++++++++++
 src/kudu/tools/tool_action_local_replica.cc | 29 +++++++++++++++++++-
 4 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index af0cbec..4363725 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -1093,6 +1093,31 @@ set<string> DataDirManager::FindTabletsByDataDirUuidIdx(int uuid_idx) const {
   return {};
 }
 
+Status DataDirManager::FindDataDirsByTabletId(const string& tablet_id,
+                                              vector<string>* data_dirs) const {
+  CHECK(data_dirs);
+  DataDirGroupPB group;
+  RETURN_NOT_OK(GetDataDirGroupPB(tablet_id, &group));
+  vector<string> data_dirs_tmp;
+  data_dirs_tmp.reserve(group.uuids_size());
+  for (const auto& uuid : group.uuids()) {
+    int uuid_idx;
+    if (!FindUuidIndexByUuid(uuid, &uuid_idx)) {
+      return Status::NotFound("unable to find index for UUID", uuid);
+    }
+    const auto* data_dir = FindDataDirByUuidIndex(uuid_idx);
+    if (!data_dir) {
+      return Status::NotFound(
+          Substitute("unable to find data dir for UUID $0 with index $1",
+                     uuid, uuid_idx));
+    }
+    data_dirs_tmp.emplace_back(data_dir->dir());
+  }
+  std::sort(data_dirs_tmp.begin(), data_dirs_tmp.end());
+  *data_dirs = std::move(data_dirs_tmp);
+  return Status::OK();
+}
+
 void DataDirManager::MarkDataDirFailedByUuid(const string& uuid) {
   int uuid_idx;
   CHECK(FindUuidIndexByUuid(uuid, &uuid_idx));
diff --git a/src/kudu/fs/data_dirs.h b/src/kudu/fs/data_dirs.h
index 2b2fc75..9bc3911 100644
--- a/src/kudu/fs/data_dirs.h
+++ b/src/kudu/fs/data_dirs.h
@@ -335,6 +335,11 @@ class DataDirManager {
   // returns a copy, returning an empty set if none are found.
   std::set<std::string> FindTabletsByDataDirUuidIdx(int uuid_idx) const;
 
+  // Returns in 'data_dirs' a sorted list of the directory names for the data
+  // dirs of the tablet specified by 'tablet_id'.
+  Status FindDataDirsByTabletId(const std::string& tablet_id,
+                                std::vector<std::string>* data_dirs) const;
+
   // ==========================================================================
   // Directory Health
   // ==========================================================================
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 522c204..e0a3a91 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -552,6 +552,7 @@ TEST_F(ToolTest, TestModeHelp) {
   {
     const vector<string> kLocalReplicaDumpModeRegexes = {
         "block_ids.*Dump the IDs of all blocks",
+        "data_dirs.*Dump the data directories",
         "meta.*Dump the metadata",
         "rowset.*Dump the rowset contents",
         "wals.*Dump all WAL"
@@ -1174,6 +1175,47 @@ TEST_F(ToolTest, TestWalDump) {
   }
 }
 
+TEST_F(ToolTest, TestLocalReplicaDumpDataDirs) {
+  const string kTestTablet = "ffffffffffffffffffffffffffffffff";
+  const string kTestTableId = "test-table";
+  const string kTestTableName = "test-fs-data-dirs-dump-table";
+  const Schema kSchema(GetSimpleTestSchema());
+  const Schema kSchemaWithIds(SchemaBuilder(kSchema).Build());
+
+  FsManagerOpts opts;
+  opts.wal_root = GetTestPath("wal");
+  opts.data_roots = {
+    GetTestPath("data0"),
+    GetTestPath("data1"),
+    GetTestPath("data2"),
+  };
+  FsManager fs(env_, opts);
+  ASSERT_OK(fs.CreateInitialFileSystemLayout());
+  ASSERT_OK(fs.Open());
+
+  pair<PartitionSchema, Partition> partition = tablet::CreateDefaultPartition(
+        kSchemaWithIds);
+  scoped_refptr<TabletMetadata> meta;
+  ASSERT_OK(TabletMetadata::CreateNew(
+      &fs, kTestTablet, kTestTableName, kTestTableId,
+      kSchemaWithIds, partition.first, partition.second,
+      tablet::TABLET_DATA_READY,
+      /*tombstone_last_logged_opid=*/ boost::none,
+      &meta));
+  string stdout;
+  NO_FATALS(RunActionStdoutString(Substitute("local_replica dump data_dirs $0 "
+                                             "--fs_wal_dir=$1 "
+                                             "--fs_data_dirs=$2",
+                                             kTestTablet, opts.wal_root,
+                                             JoinStrings(opts.data_roots, ",")),
+                                  &stdout));
+  vector<string> expected;
+  for (const auto& data_root : opts.data_roots) {
+    expected.emplace_back(JoinPathSegments(data_root, "data"));
+  }
+  ASSERT_EQ(JoinStrings(expected, "\n"), stdout);
+}
+
 TEST_F(ToolTest, TestLocalReplicaDumpMeta) {
   const string kTestDir = GetTestPath("test");
   const string kTestTablet = "ffffffffffffffffffffffffffffffff";
diff --git a/src/kudu/tools/tool_action_local_replica.cc b/src/kudu/tools/tool_action_local_replica.cc
index 21edb5e..2173acb 100644
--- a/src/kudu/tools/tool_action_local_replica.cc
+++ b/src/kudu/tools/tool_action_local_replica.cc
@@ -46,6 +46,7 @@
 #include "kudu/consensus/opid.pb.h"
 #include "kudu/fs/block_id.h"
 #include "kudu/fs/block_manager.h"
+#include "kudu/fs/data_dirs.h"
 #include "kudu/fs/fs_manager.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/ref_counted.h"
@@ -681,7 +682,23 @@ Status DumpMeta(const RunnerContext& context) {
   unique_ptr<FsManager> fs_manager;
   RETURN_NOT_OK(FsInit(&fs_manager));
   const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
-  RETURN_NOT_OK(DumpTabletMeta(fs_manager.get(), tablet_id, 0));
+  return DumpTabletMeta(fs_manager.get(), tablet_id, 0);
+}
+
+Status DumpDataDirs(const RunnerContext& context) {
+  unique_ptr<FsManager> fs_manager;
+  RETURN_NOT_OK(FsInit(&fs_manager));
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
+  // Load the tablet meta to make sure the tablet's data directories are loaded
+  // into the manager.
+  scoped_refptr<TabletMetadata> unused_meta;
+  RETURN_NOT_OK(TabletMetadata::Load(fs_manager.get(), tablet_id, &unused_meta));
+  vector<string> data_dirs;
+  RETURN_NOT_OK(fs_manager->dd_manager()->FindDataDirsByTabletId(tablet_id,
+                                                                 &data_dirs));
+  for (const auto& dir : data_dirs) {
+    cout << dir << endl;
+  }
   return Status::OK();
 }
 
@@ -695,6 +712,15 @@ unique_ptr<Mode> BuildDumpMode() {
       .AddOptionalParameter("fs_wal_dir")
       .Build();
 
+  unique_ptr<Action> dump_data_dirs =
+      ActionBuilder("data_dirs", &DumpDataDirs)
+      .Description("Dump the data directories where the replica's data is stored")
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
+      .AddOptionalParameter("fs_data_dirs")
+      .AddOptionalParameter("fs_metadata_dir")
+      .AddOptionalParameter("fs_wal_dir")
+      .Build();
+
   unique_ptr<Action> dump_meta =
       ActionBuilder("meta", &DumpMeta)
       .Description("Dump the metadata of a local replica")
@@ -733,6 +759,7 @@ unique_ptr<Mode> BuildDumpMode() {
   return ModeBuilder("dump")
       .Description("Dump a Kudu filesystem")
       .AddAction(std::move(dump_block_ids))
+      .AddAction(std::move(dump_data_dirs))
       .AddAction(std::move(dump_meta))
       .AddAction(std::move(dump_rowset))
       .AddAction(std::move(dump_wals))


[kudu] 03/03: docs: fix two broken links in the installation doc

Posted by wd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 794a9ecf8a23652ab8d5f65b4536f3850de1d445
Author: Greg Solovyev <fi...@gmail.com>
AuthorDate: Wed Feb 13 18:20:33 2019 -0800

    docs: fix two broken links in the installation doc
    
    Change-Id: I249df2ac3a88bda9825419a852a752423a5cb04f
    Reviewed-on: http://gerrit.cloudera.org:8080/12480
    Tested-by: Kudu Jenkins
    Reviewed-by: Grant Henke <gr...@apache.org>
---
 docs/installation.adoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/installation.adoc b/docs/installation.adoc
index 443bedb..7d219f2 100644
--- a/docs/installation.adoc
+++ b/docs/installation.adoc
@@ -34,7 +34,7 @@ from source. To run Kudu without installing anything, use the link:quickstart.ht
 NOTE: Cloudera Manager users can install and manage Kudu with
 link:http://www.cloudera.com/content/www/en-us/products/cloudera-manager.html[Cloudera Manager],
 instead of following these instructions. For more information, see Cloudera's
-link:http://www.cloudera.com/documentation/kudu/latest/topics/kudu_installation.html[Kudu documentation].
+link:http://www.cloudera.com/documentation/enterprise/latest/topics/kudu_install_upgrade_intro.html[Kudu documentation].
 
 .Upgrading Kudu
 To upgrade Kudu from a previous version, see <<upgrade>>.
@@ -573,7 +573,7 @@ $ sudo xcode-select --install
 .macOS Known Issues
 ====
 Kudu support for macOS is experimental, and should only be used for development.
-See link:https://issues.apache.org/browse/KUDU-1219[macOS Limitations & Known Issues]
+See link:https://issues.apache.org/jira/browse/KUDU-1219[macOS Limitations & Known Issues]
 for more information.
 ====