You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/10/09 17:43:27 UTC
[1/2] kudu git commit: [webui] KUDU-844 and other
/tablet-rowsetlayout-svg improvements
Repository: kudu
Updated Branches:
refs/heads/master 9ec8d28db -> b193e3786
[webui] KUDU-844 and other /tablet-rowsetlayout-svg improvements
This patch addresses KUDU-844, an issue where rowsets unavailable for
compaction weren't shown in the rowset SVG layout. These rowsets aren't
displayed because the rowset SVG is built from compaction policy output,
and the compaction policy obviously does not consider rowsets that
aren't available for compaction. There are two kinds of rowsets that
will be excluded: the memrowset, which doesn't belong in the SVG output,
and rowsets currently being compacted. The latter are tricky to
integrate into the SVG because there is a race with the compaction
finishing, and after a compaction the rowset may no longer be
relevant (merge compaction), or its bounds may have changed (major delta
compaction). Working around this problem would be a lot of effort, just
to add dubiously useful information to the layout diagram. Instead, I
opted just to disclose the issue: there's a note saying how many rowsets
are currently being compacted included under the layout diagram.
I also made a couple of quality of life improvements to the rowset
layout page.
1. I removed the size label from the rowsets rectangles in the SVG. They
didn't add so much individually, but they did make it really hard to
see what was going in once there were a lot of rowsets.
2. I added a table with the 5-number summary of rowset sizes, to make up
for taking away the individual sizes in the diagram. I chose the
summary because in my experience it's the best way to get a quick
idea what a distribution looks like (better than, e.g., the mean and
standard deviation.)
Here's the original way a busy layout diagram looks with the sizes:
https://github.com/wdberkeley/kudu/blob/svg_screenshots/layout.png
And here's how it looks without them:
https://github.com/wdberkeley/kudu/blob/svg_screenshots/layout_no_text.png
Here's an example summary table:
https://github.com/wdberkeley/kudu/blob/svg_screenshots/new_layout_page.png
Finally, I added some <tr> pairs that were missing from the /tablets
template.
Change-Id: I7475f14093a187fde1329546b10c314da49fe08a
Reviewed-on: http://gerrit.cloudera.org:8080/11613
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/91faf2a1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/91faf2a1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/91faf2a1
Branch: refs/heads/master
Commit: 91faf2a1205e32e9d8293e263300718499bdf575
Parents: 9ec8d28
Author: Will Berkeley <wd...@gmail.org>
Authored: Sun Oct 7 22:49:58 2018 -0700
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Tue Oct 9 17:37:53 2018 +0000
----------------------------------------------------------------------
src/kudu/tablet/svg_dump.cc | 4 ---
src/kudu/tablet/tablet.cc | 70 ++++++++++++++++++++++++++++++++++++++--
www/tablets.mustache | 4 +--
3 files changed, 69 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/91faf2a1/src/kudu/tablet/svg_dump.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/svg_dump.cc b/src/kudu/tablet/svg_dump.cc
index 9c88655..93a3cc2 100644
--- a/src/kudu/tablet/svg_dump.cc
+++ b/src/kudu/tablet/svg_dump.cc
@@ -136,10 +136,6 @@ void DumpSVG(const vector<RowSetInfo>& candidates,
R"(<rect x="$0" y="$1" width="$2" height="$3" stroke="#000" fill="$4"/>)",
x, y, width, kRowHeight, color)
<< endl;
- out << Substitute(R"+(<text x="$0" y="$1" width="$2" height="$3" )+"
- R"+(fill="rgb(0,0,0)">$4MB</text>)+",
- x, y + kRowHeight, width, kRowHeight, cand->size_mb())
- << endl;
}
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/91faf2a1/src/kudu/tablet/tablet.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 05a024f..0a7a59a 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -172,6 +172,7 @@ using kudu::MaintenanceManager;
using kudu::clock::HybridClock;
using kudu::fs::IOContext;
using kudu::log::LogAnchorRegistry;
+using std::endl;
using std::ostream;
using std::pair;
using std::shared_ptr;
@@ -2299,13 +2300,76 @@ void Tablet::PrintRSLayout(ostream* o) {
RowSetInfo::CollectOrdered(*rowsets_copy, &min, &max);
DumpCompactionSVG(min, picked, o, /*print_xml_header=*/false);
- *o << "<h2>Compaction policy log</h2>" << std::endl;
+ // Compaction policy ignores rowsets unavailable for compaction. This is good,
+ // except it causes the SVG to be potentially missing rowsets. It's hard to
+ // take these presently-compacting rowsets into account because we are racing
+ // against the compaction finishing, and at the end of the compaction the
+ // rowsets might no longer exist (merge compaction) or their bounds may have
+ // changed (major delta compaction). So, let's just disclose how many of these
+ // rowsets there are.
+ int num_rowsets_unavailable_for_compaction = std::count_if(
+ rowsets_copy->all_rowsets().begin(),
+ rowsets_copy->all_rowsets().end(),
+ [](const shared_ptr<RowSet>& rowset) {
+ // The first condition excludes the memrowset.
+ return rowset->metadata() && !rowset->IsAvailableForCompaction();
+ });
+ *o << Substitute("<div><p>In addition to the rowsets pictured and listed, "
+ "there are $0 rowset(s) currently undergoing compactions."
+ "</p></div>",
+ num_rowsets_unavailable_for_compaction)
+ << endl;
+
+ // Compute some summary statistics for the tablet's rowsets.
+ const auto num_rowsets = min.size();
+ if (num_rowsets > 0) {
+ vector<int64_t> rowset_sizes;
+ rowset_sizes.reserve(num_rowsets);
+ for (const auto& rsi : min) {
+ rowset_sizes.push_back(rsi.size_bytes());
+ }
+ *o << "<table class=\"table tablet-striped table-hover\">" << endl;
+ // Compute the stats quick'n'dirty by sorting and looking at approximately
+ // the right spot.
+ // TODO(wdberkeley): Could use an O(n) quickselect-based algorithm.
+ // TODO(wdberkeley): A bona fide box-and-whisker plot would be nice.
+ // d3.js can make really nice ones: https://bl.ocks.org/mbostock/4061502.
+ std::sort(rowset_sizes.begin(), rowset_sizes.end());
+ const auto size_bytes_min = rowset_sizes[0];
+ const auto size_bytes_first_quartile = rowset_sizes[num_rowsets / 4];
+ const auto size_bytes_median = rowset_sizes[num_rowsets / 2];
+ const auto size_bytes_third_quartile = rowset_sizes[3 * num_rowsets / 4];
+ const auto size_bytes_max = rowset_sizes[num_rowsets - 1];
+ *o << Substitute("<thead><tr>"
+ " <th>Statistic</th>"
+ " <th>Approximate Value</th>"
+ "<tr></thead>"
+ "<tbody>"
+ " <tr><td>Count</td><td>$0</td></tr>"
+ " <tr><td>Min</td><td>$1</td></tr>"
+ " <tr><td>First quartile</td><td>$2</td></tr>"
+ " <tr><td>Median</td><td>$3</td></tr>"
+ " <tr><td>Third quartile</td><td>$4</td></tr>"
+ " <tr><td>Max</td><td>$5</td></tr>"
+ "<tbody>",
+ num_rowsets,
+ HumanReadableNumBytes::ToString(size_bytes_min),
+ HumanReadableNumBytes::ToString(size_bytes_first_quartile),
+ HumanReadableNumBytes::ToString(size_bytes_median),
+ HumanReadableNumBytes::ToString(size_bytes_third_quartile),
+ HumanReadableNumBytes::ToString(size_bytes_max));
+ *o << "</table>" << endl;
+ }
+
+ // TODO(wdberkeley): Should we even display this? It's one line per rowset
+ // and doesn't contain any useful information except each rowset's size.
+ *o << "<h2>Compaction policy log</h2>" << endl;
*o << "<pre>" << std::endl;
for (const string& s : log) {
- *o << EscapeForHtmlToString(s) << std::endl;
+ *o << EscapeForHtmlToString(s) << endl;
}
- *o << "</pre>" << std::endl;
+ *o << "</pre>" << endl;
}
string Tablet::LogPrefix() const {
http://git-wip-us.apache.org/repos/asf/kudu/blob/91faf2a1/www/tablets.mustache
----------------------------------------------------------------------
diff --git a/www/tablets.mustache b/www/tablets.mustache
index 886239b..ea1f94a 100644
--- a/www/tablets.mustache
+++ b/www/tablets.mustache
@@ -26,7 +26,7 @@ There are no tablet replicas.
<thead><tr><th>Status</th><th>Count</th><th>Percentage</th></tr></thead>
<tbody>
{{#statuses}}
- <td>{{status}}</td><td>{{count}}</td><td>{{percentage}}</td>
+ <tr><td>{{status}}</td><td>{{count}}</td><td>{{percentage}}</td></tr>
{{/statuses}}
</tbody>
<tfoot><tr><td>Total</td><td>{{total_count}}</td><td></td></tfoot>
@@ -70,7 +70,7 @@ There are no tablet replicas.
<thead><tr><th>Status</th><th>Count</th><th>Percentage</th></tr></thead>
<tbody>
{{#replica_statuses}}
- <td>{{status}}</td><td>{{count}}</td><td>{{percentage}}</td>
+ <tr><td>{{status}}</td><td>{{count}}</td><td>{{percentage}}</td></tr>
{{/replica_statuses}}
</tbody>
<tfoot><tr><td>Total</td><td>{{total_count}}</td><td></td></tfoot>
[2/2] kudu git commit: [tests] location assignment for
ExternaMiniCluster
Posted by al...@apache.org.
[tests] location assignment for ExternaMiniCluster
Introduced location assignment rules for tablet servers running as
part of ExternalMiniCluster. With this changelist, it's possible
to have tablet servers of ExternalMiniCluster to be spread among
locations in accordance with simple number-of-servers-at-location
mapping rules.
For example, the following is a self-descriptive example of such
mapping rules set to spread six tablet servers in a cluster among
three locations:
L0:1 L1:2 L2:3
Added an integration test to verify the location assignment
rules for ExternalMiniCluster work as intended.
Change-Id: I63309804cf2fdd8a620b50abdd7133af6a033c30
Reviewed-on: http://gerrit.cloudera.org:8080/11606
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wd...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b193e378
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b193e378
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b193e378
Branch: refs/heads/master
Commit: b193e3786861686590b6bf67bc507d8d4d18d8e3
Parents: 91faf2a
Author: Alexey Serbin <as...@cloudera.com>
Authored: Fri Oct 5 22:12:41 2018 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Tue Oct 9 17:42:50 2018 +0000
----------------------------------------------------------------------
src/kudu/integration-tests/CMakeLists.txt | 2 +
.../integration-tests/cluster_itest_util.cc | 1 +
src/kudu/integration-tests/cluster_itest_util.h | 1 +
.../scripts/assign-location.py | 244 +++++++++++++++++++
.../ts_location_assignment-itest.cc | 132 ++++++++++
src/kudu/master/ts_descriptor.cc | 20 +-
src/kudu/mini-cluster/external_mini_cluster.cc | 24 +-
src/kudu/mini-cluster/external_mini_cluster.h | 10 +
8 files changed, 428 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index 512a454..7ecb697 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -114,6 +114,8 @@ ADD_KUDU_TEST(tombstoned_voting-imc-itest)
ADD_KUDU_TEST(tombstoned_voting-itest)
ADD_KUDU_TEST(tombstoned_voting-stress-test RUN_SERIAL true)
ADD_KUDU_TEST(token_signer-itest)
+ADD_KUDU_TEST(ts_location_assignment-itest
+ DATA_FILES scripts/assign-location.py)
ADD_KUDU_TEST(ts_recovery-itest PROCESSORS 4)
ADD_KUDU_TEST(ts_tablet_manager-itest)
ADD_KUDU_TEST(update_scan_delta_compact-test RUN_SERIAL true)
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/cluster_itest_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/cluster_itest_util.cc b/src/kudu/integration-tests/cluster_itest_util.cc
index 7231094..186cf01 100644
--- a/src/kudu/integration-tests/cluster_itest_util.cc
+++ b/src/kudu/integration-tests/cluster_itest_util.cc
@@ -308,6 +308,7 @@ Status CreateTabletServerMap(const shared_ptr<MasterServiceProxy>& master_proxy,
unique_ptr<TServerDetails> peer(new TServerDetails);
peer->instance_id.CopyFrom(entry.instance_id());
peer->registration.CopyFrom(entry.registration());
+ peer->location = entry.location();
CreateTsClientProxies(addresses[0],
messenger,
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/cluster_itest_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/cluster_itest_util.h b/src/kudu/integration-tests/cluster_itest_util.h
index c5ba53a..1768880 100644
--- a/src/kudu/integration-tests/cluster_itest_util.h
+++ b/src/kudu/integration-tests/cluster_itest_util.h
@@ -79,6 +79,7 @@ struct TServerDetails {
std::unique_ptr<tserver::TabletServerAdminServiceProxy> tserver_admin_proxy;
std::unique_ptr<consensus::ConsensusServiceProxy> consensus_proxy;
std::unique_ptr<server::GenericServiceProxy> generic_proxy;
+ std::string location;
// Convenience function to get the UUID from the instance_id struct.
const std::string& uuid() const;
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/scripts/assign-location.py
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/scripts/assign-location.py b/src/kudu/integration-tests/scripts/assign-location.py
new file mode 100644
index 0000000..5714772
--- /dev/null
+++ b/src/kudu/integration-tests/scripts/assign-location.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import errno
+import fcntl
+import json
+import time
+import random
+
+# This is a simple sequencer to be run as a location assignment script
+# by a Kudu master. The script can be used in location-aware test scenarios
+# and other cases when location assignment rules are specified simply as the
+# distribution of tablet servers among locations: i.e. how many tablet
+# servers should be in every specified location (see below for an example).
+#
+# The script takes as input location mapping rules and an identifier.
+# On success, the script prints the location assigned to the specified
+# identifier to stdout. The identifier might be any string uniquely identifying
+# a tablet server.
+#
+# Locations are assigned based on:
+# a) Location mapping rules specified in the command line and sequencer's
+# offset persistently stored in a state file.
+# b) Previously established and persisted { id, location } mappings in the
+# state file.
+#
+# Once assigned, the location for the specified identifier is recorded and
+# output again upon next call of the script for the same identifier.
+#
+# It's safe to run multiple instances of the script concurrently with the
+# same set of parameters. The access to the sequencer's state file is
+# serialized and the scripts produces consistent results for all concurrent
+# callers.
+#
+# A location mapping rule is specified as a pair 'loc:num', where the 'num'
+# stands for the number of servers to assign to the location 'loc'. Location
+# mapping rules are provided to the script by --map 'loc:num' command line
+# arguments.
+#
+# Below is an example of invocation of the script for location mapping rules
+# specifying that location 'l0' should have one tablet server, location 'l1'
+# should have two, and location 'l2' should have three. The script is run
+# to assign a location for a tablet server running at IP address 127.1.2.3.
+#
+# assign-location.py --map l0:1 --map l1:2 --map l2:3 127.1.2.3
+#
+
+class LocationAssignmentRule(object):
+ def __init__(self, location_mapping_rules):
+ # Convert the input location information into an auxiliary array of
+ # location strings.
+ self.location_mapping_rules = location_mapping_rules
+ if self.location_mapping_rules is None:
+ self.location_mapping_rules = []
+ self.locations = []
+ self.total_count = 0
+
+ seen_locations = []
+ for info in self.location_mapping_rules:
+ location, server_num_str = info.split(':')
+ seen_locations.append(location)
+ server_num = int(server_num_str)
+ for i in range(0, server_num):
+ self.total_count += 1
+ self.locations.append(location)
+ assert (len(set(seen_locations)) == len(seen_locations)), \
+ 'duplicate locations specified: {}'.format(seen_locations)
+
+ def get_location(self, idx):
+ """
+ Get location for the specified index.
+ """
+ if self.locations:
+ return self.locations[idx % len(self.locations)]
+ else:
+ return ""
+
+
+def acquire_advisory_lock(fpath):
+ """
+ Acquire a lock on a special .lock file. Don't block while trying: return
+ if failed to acquire a lock in 30 seconds.
+ """
+ timeout_seconds = 30
+ now = time.clock()
+ deadline = now + timeout_seconds
+ random.seed(int(now))
+ fpath_lock_file = fpath + ".lock"
+ # Open the lock file; create the file if doesn't exist.
+ lock_file = open(fpath_lock_file, 'w+')
+ got_lock = False
+ while time.clock() < deadline:
+ try:
+ fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ got_lock = True
+ break
+ except IOError as e:
+ if e.errno != errno.EAGAIN:
+ raise
+ else:
+ time.sleep(random.uniform(0.001, 0.100))
+
+ if not got_lock:
+ raise Exception('could not obtain exclusive lock for {} in {} seconds',
+ fpath_lock_file, timeout_seconds)
+
+ return lock_file
+
+
+def get_location(fpath, rule, uid, relaxed):
+ """
+ Return location for the specified identifier 'uid'. To do that, use the
+ specified location mapping rules and the information stored
+ in the sequencer's state file.
+
+ * Obtain advisory lock for the state file (using additional .lock file)
+ * If the sequencer's state file exists:
+ 1. Open the state file in read-only mode.
+ 2. Read the information from the state file and search for location
+ assigned to the server with the specified identifier.
+ a. If already assigned location found:
+ -- Return the location.
+ b. If location assigned to the identifier is not found:
+ -- Use current sequence number 'seq' to assign next location
+ by calling LocationAssignmentRule.get_location(seq).
+ -- Add the newly generated location assignment into the
+ sequencer's state.
+ -- Increment the sequence number.
+ -- Reopen the state file for writing (if file exists)
+ -- Rewrite the file with the new state of the sequencer.
+ -- Return the newly assigned location.
+ * If the sequencer's state file does not exist:
+ 1. Set sequence number 'seq' to 0.
+ 2. Use current sequence number 'seq' to assign next location
+ by calling LocationAssignmentRule.get_location(seq).
+ 3. Update the sequencer's state accordingly.
+ 3. Rewrite the file with the new state of the sequencer.
+ 4. Return the newly assigned location.
+ """
+ lock_file = acquire_advisory_lock(fpath)
+ state_file = None
+ try:
+ state_file = open(fpath)
+ except IOError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ new_assignment = False
+ if state_file is None:
+ seq = 0
+ state = {}
+ state['seq'] = seq
+ state['mapping_rules'] = rule.location_mapping_rules
+ state['mappings'] = {}
+ mappings = state['mappings']
+ new_assignment = True
+ else:
+ # If the file exists, it must have proper content.
+ state = json.load(state_file)
+ seq = state.get('seq')
+ mapping_rules = state.get('mapping_rules')
+ # Make sure the stored mapping rule corresponds to the specified in args.
+ rule_stored = json.dumps(mapping_rules)
+ rule_specified = json.dumps(rule.location_mapping_rules)
+ if rule_stored != rule_specified:
+ raise Exception('stored and specified mapping rules mismatch: '
+ '{} vs {}'.format(rule_stored, rule_specified))
+ mappings = state['mappings']
+ location = mappings.get(uid, None)
+ if location is None:
+ seq += 1
+ state['seq'] = seq
+ new_assignment = True
+
+ if not new_assignment:
+ return location
+
+ if not relaxed and rule.total_count != 0 and rule.total_count <= seq:
+ raise Exception('too many unique identifiers ({}) to assign next location '
+ 'using mapping rules {}'.format(
+ seq + 1, rule.location_mapping_rules))
+
+ if relaxed and rule.total_count <= seq:
+ return ""
+
+ # Get next location and add the { uid, location} binding into the mappings.
+ location = rule.get_location(seq)
+ mappings[uid] = location
+
+ # Rewrite the file with the updated state information.
+ if state_file is not None:
+ state_file.close()
+ state_file = open(fpath, 'w+')
+ json.dump(state, state_file)
+ state_file.close()
+ lock_file.close()
+ return location
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--state_store",
+ nargs="?",
+ default="/tmp/location-sequencer-state",
+ help="path to a file to store the sequencer's state")
+ parser.add_argument("--map", "-m",
+ action="append",
+ dest="location_mapping_rules",
+ metavar="RULE",
+ help="location mapping rule: number of tablet servers per specified "
+ "location in form <location>:<number>; this option may be specified "
+ "multiple times")
+ parser.add_argument("--relaxed",
+ action="store_true",
+ help="whether to allow more location assignments than specified "
+ "by the specified mapping rules")
+ parser.add_argument("uid",
+ help="hostname, IP address, or any other unique identifier")
+ args = parser.parse_args()
+
+ location = get_location(args.state_store,
+ LocationAssignmentRule(args.location_mapping_rules), args.uid, args.relaxed)
+ print(location)
+
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/ts_location_assignment-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/ts_location_assignment-itest.cc b/src/kudu/integration-tests/ts_location_assignment-itest.cc
new file mode 100644
index 0000000..96b3dcd
--- /dev/null
+++ b/src/kudu/integration-tests/ts_location_assignment-itest.cc
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <ostream>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/stl_util.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/integration-tests/cluster_itest_util.h"
+#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/util/random.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+using kudu::cluster::ExternalMiniCluster;
+using kudu::cluster::ExternalMiniClusterOptions;
+using kudu::cluster::LocationInfo;
+using kudu::itest::TServerDetails;
+using std::string;
+using std::unique_ptr;
+using std::unordered_map;
+using strings::Substitute;
+
+namespace kudu {
+
+class TsLocationAssignmentITest :
+ public KuduTest,
+ public ::testing::WithParamInterface<std::tuple<int, int>> {
+ public:
+ TsLocationAssignmentITest()
+ : rng_(SeedRandom()) {
+ const auto& param = GetParam();
+ opts_.num_masters = std::get<0>(param);
+ opts_.num_tablet_servers = std::get<1>(param);
+ }
+
+ virtual ~TsLocationAssignmentITest() = default;
+
+ protected:
+ void StartCluster() {
+ // Generate random location mapping.
+ LocationInfo info;
+ int num_mappings_left = opts_.num_tablet_servers;
+ int loc_idx = 0;
+ while (true) {
+ auto location = Substitute("/L$0", loc_idx);
+ if (num_mappings_left <= 1) {
+ EmplaceOrDie(&info, std::move(location), 1);
+ break;
+ }
+ const int num = static_cast<int>(rng_.Uniform(num_mappings_left));
+ if (num == 0) {
+ continue;
+ }
+ EmplaceOrDie(&info, std::move(location), num);
+
+ num_mappings_left -= num;
+ ++loc_idx;
+ }
+
+ opts_.location_info = std::move(info);
+ cluster_.reset(new ExternalMiniCluster(opts_));
+ ASSERT_OK(cluster_->Start());
+ }
+
+ void CheckLocationInfo() {
+ unordered_map<string, itest::TServerDetails*> ts_map;
+ ASSERT_OK(itest::CreateTabletServerMap(cluster_->master_proxy(0),
+ cluster_->messenger(),
+ &ts_map));
+ ValueDeleter deleter(&ts_map);
+
+ LocationInfo location_info;
+ for (const auto& desc : ts_map) {
+ ++LookupOrEmplace(&location_info, desc.second->location, 0);
+ }
+ ASSERT_EQ(opts_.location_info, location_info);
+ }
+
+ ThreadSafeRandom rng_;
+ ExternalMiniClusterOptions opts_;
+ unique_ptr<cluster::ExternalMiniCluster> cluster_;
+};
+
+// Verify that the location assignment works as expected for tablet servers
+// run as part of ExternalMiniCluster. Also verify that every tablet server
+// is assigned the same location after restart once the location assignment
+// script is kept the same between restarts.
+TEST_P(TsLocationAssignmentITest, Basic) {
+ if (!AllowSlowTests()) {
+ LOG(WARNING) << "test is skipped; set KUDU_ALLOW_SLOW_TESTS=1 to run";
+ return;
+ }
+
+ NO_FATALS(StartCluster());
+ NO_FATALS(CheckLocationInfo());
+ NO_FATALS(cluster_->AssertNoCrashes());
+
+ cluster_->Shutdown();
+
+ ASSERT_OK(cluster_->Restart());
+ NO_FATALS(CheckLocationInfo());
+ NO_FATALS(cluster_->AssertNoCrashes());
+}
+
+INSTANTIATE_TEST_CASE_P(, TsLocationAssignmentITest,
+ ::testing::Combine(::testing::Values(1, 3),
+ ::testing::Values(1, 8, 16, 32)));
+
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/master/ts_descriptor.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/ts_descriptor.cc b/src/kudu/master/ts_descriptor.cc
index e3c7d76..ae5398a 100644
--- a/src/kudu/master/ts_descriptor.cc
+++ b/src/kudu/master/ts_descriptor.cc
@@ -60,9 +60,14 @@ DEFINE_string(location_mapping_cmd, "",
"using location awareness features this flag should not be set.");
TAG_FLAG(location_mapping_cmd, evolving);
+DEFINE_bool(location_mapping_by_uuid, false,
+ "Whether the location command is given tablet server identifier "
+ "instead of hostname/IP address (for tests only).");
+TAG_FLAG(location_mapping_by_uuid, hidden);
+TAG_FLAG(location_mapping_by_uuid, unsafe);
+
using kudu::pb_util::SecureDebugString;
using kudu::pb_util::SecureShortDebugString;
-using std::make_shared;
using std::shared_ptr;
using std::string;
using std::vector;
@@ -223,13 +228,18 @@ Status TSDescriptor::Register(const NodeInstancePB& instance,
// mapping script.
const string& location_mapping_cmd = FLAGS_location_mapping_cmd;
if (!location_mapping_cmd.empty()) {
- const auto& host = registration_->rpc_addresses(0).host();
+ // In some test scenarios the location is assigned per tablet server UUID.
+ // That's the case when multiple (or even all) tablet servers have the same
+ // IP address for their RPC endpoint.
+ const auto& cmd_arg = FLAGS_location_mapping_by_uuid
+ ? permanent_uuid() : registration_->rpc_addresses(0).host();
+ TRACE(Substitute("tablet server $0: assigning location", permanent_uuid()));
string location;
- TRACE("Assigning location");
Status s = GetLocationFromLocationMappingCmd(location_mapping_cmd,
- host,
+ cmd_arg,
&location);
- TRACE("Assigned location");
+ TRACE(Substitute(
+ "tablet server $0: assigned location '$1'", permanent_uuid(), location));
// Assign the location under the lock if location resolution succeeds. If
// it fails, log the error.
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/mini-cluster/external_mini_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/mini-cluster/external_mini_cluster.cc b/src/kudu/mini-cluster/external_mini_cluster.cc
index 414956d..f86294c 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.cc
+++ b/src/kudu/mini-cluster/external_mini_cluster.cc
@@ -223,7 +223,6 @@ Status ExternalMiniCluster::Start() {
return Status::OK();
}
-
void ExternalMiniCluster::ShutdownNodes(ClusterNodes nodes) {
if (nodes == ClusterNodes::ALL || nodes == ClusterNodes::TS_ONLY) {
for (const scoped_refptr<ExternalTabletServer>& ts : tablet_servers_) {
@@ -377,6 +376,29 @@ Status ExternalMiniCluster::StartMasters() {
flags.emplace_back(Substitute("--master_addresses=$0",
HostPort::ToCommaSeparatedString(master_rpc_addrs)));
}
+ if (!opts_.location_info.empty()) {
+ string bin_path;
+ RETURN_NOT_OK(DeduceBinRoot(&bin_path));
+ const auto mapping_script_path =
+ JoinPathSegments(bin_path, "scripts/assign-location.py");
+ const auto state_store_fpath =
+ JoinPathSegments(opts_.cluster_root, "location-assignment.state");
+ auto location_cmd = Substitute("$0 --state_store=$1",
+ mapping_script_path, state_store_fpath);
+ for (const auto& elem : opts_.location_info) {
+ // Per-location mapping rule specified as a pair 'location:num_servers',
+ // where 'location' is the location string and 'num_servers' is the number
+ // of tablet servers to be assigned the location.
+ location_cmd += Substitute(" --map $0:$1", elem.first, elem.second);
+ }
+ flags.emplace_back(Substitute("--location_mapping_cmd=$0", location_cmd));
+# if defined(__APPLE__)
+ // On macOS, it's not possible to have unique loopback interfaces. To make
+ // location mapping working, a tablet server is identified by its UUID
+ // instead of IP address of its RPC end-point.
+ flags.emplace_back("--location_mapping_by_uuid");
+# endif
+ }
string exe = GetBinaryPath(kMasterBinaryName);
// Start the masters.
http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/mini-cluster/external_mini_cluster.h
----------------------------------------------------------------------
diff --git a/src/kudu/mini-cluster/external_mini_cluster.h b/src/kudu/mini-cluster/external_mini_cluster.h
index b3f3a4b..6244887 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.h
+++ b/src/kudu/mini-cluster/external_mini_cluster.h
@@ -76,6 +76,9 @@ class ExternalDaemon;
class ExternalMaster;
class ExternalTabletServer;
+// Location --> number of tablet servers in location.
+typedef std::map<std::string, int> LocationInfo;
+
struct ExternalMiniClusterOptions {
ExternalMiniClusterOptions();
@@ -160,6 +163,13 @@ struct ExternalMiniClusterOptions {
//
// Default: 3 seconds.
MonoDelta rpc_negotiation_timeout;
+
+ // Parameter to specify the layout of tablet servers across cluster locations
+ // in form of pairs { location, num_tablet_servers }. The empty container
+ // means no locations are configured for the cluster.
+ //
+ // Default: empty
+ LocationInfo location_info;
};
// A mini-cluster made up of subprocesses running each of the daemons