You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2018/10/16 20:59:16 UTC
[1/2] kudu git commit: [compaction] KUDU-2056: Expose a metric for
how much a tablet needs to be compacted
Repository: kudu
Updated Branches:
refs/heads/master 8c184e2a9 -> 09848055b
[compaction] KUDU-2056: Expose a metric for how much a tablet needs to be compacted
This adds a new metric 'average_diskrowset_height' that reflects how
uncompacted a tablet replica is. This metric is obtained by integrating
the height function with respect to the by-data-size probability
distribution used by the compaction policy.
To implement the integration, I piggy-backed on the function that
computes the CDF for the rowset layout, since computing the integral
requires computing the CDF and it seemed wasteful to first compute the
CDF, then go through an almost entirely similar bit of logic to compute
the average height.
Change-Id: I98493b901d37bb278167ba2fe98d322a86a1f0f9
Reviewed-on: http://gerrit.cloudera.org:8080/11639
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d3684a7b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d3684a7b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d3684a7b
Branch: refs/heads/master
Commit: d3684a7b2add8f06b7189adb9ce9222b8ae1eff5
Parents: 8c184e2
Author: Will Berkeley <wd...@gmail.org>
Authored: Mon Oct 8 16:09:41 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Tue Oct 16 20:47:13 2018 +0000
----------------------------------------------------------------------
src/kudu/tablet/compaction_policy-test.cc | 166 +++++++++++++++++++++++++
src/kudu/tablet/compaction_policy.cc | 2 +-
src/kudu/tablet/rowset_info.cc | 101 +++++++++------
src/kudu/tablet/rowset_info.h | 19 +--
src/kudu/tablet/tablet.cc | 33 ++++-
src/kudu/tablet/tablet_metrics.cc | 10 +-
src/kudu/tablet/tablet_metrics.h | 9 +-
7 files changed, 287 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/compaction_policy-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/compaction_policy-test.cc b/src/kudu/tablet/compaction_policy-test.cc
index 4277f9d..8ec0829 100644
--- a/src/kudu/tablet/compaction_policy-test.cc
+++ b/src/kudu/tablet/compaction_policy-test.cc
@@ -21,6 +21,7 @@
#include <string>
#include <unordered_set>
#include <vector>
+#include <utility>
#include <glog/logging.h>
#include <glog/stl_logging.h>
@@ -33,6 +34,7 @@
#include "kudu/tablet/compaction_policy.h"
#include "kudu/tablet/mock-rowsets.h"
#include "kudu/tablet/rowset.h"
+#include "kudu/tablet/rowset_info.h"
#include "kudu/tablet/rowset_tree.h"
#include "kudu/util/env.h"
#include "kudu/util/faststring.h"
@@ -276,5 +278,169 @@ TEST_F(TestCompactionPolicy, KUDU2251) {
ASSERT_LT(quality, 2.0);
}
+namespace {
+double ComputeAverageRowsetHeight(
+ const vector<std::pair<string, string>>& intervals) {
+ RowSetVector rowsets;
+ for (const auto& interval : intervals) {
+ rowsets.push_back(std::make_shared<MockDiskRowSet>(interval.first,
+ interval.second));
+ }
+ RowSetTree tree;
+ CHECK_OK(tree.Reset(rowsets));
+
+ double avg_height;
+ RowSetInfo::ComputeCdfAndCollectOrdered(tree, &avg_height, nullptr, nullptr);
+ return avg_height;
+}
+} // anonymous namespace
+
+class KeySpaceCdfTest : public KuduTest {
+ protected:
+ static void AssertWithinEpsilon(double epsilon,
+ double expected,
+ double actual) {
+ ASSERT_GE(actual, expected - epsilon);
+ ASSERT_LE(actual, expected + epsilon);
+ }
+};
+
+// Test the computation of average rowset heights. This isn't strictly used in
+// compaction policy, but this is a convenient place to put it, for now.
+TEST_F(KeySpaceCdfTest, TestComputingAverageRowSetHeight) {
+ // No rowsets.
+ EXPECT_EQ(0.0, ComputeAverageRowsetHeight({ }));
+
+ /* A rowset that's one key wide.
+ * |
+ */
+ EXPECT_EQ(0.0, ComputeAverageRowsetHeight({ { "A", "A" } }));
+
+ /* A single rowset.
+ * [ --- ]
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" } }));
+
+ /* Two rowsets with no empty space between.
+ * [ --- ][ --- ]
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" }, { "B", "C" } }));
+
+
+ /* Three rowsets with no empty spaces between.
+ * [ --- ][ --- ][ --- ]
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" },
+ { "B", "C" },
+ { "C", "D" } }));
+
+ /* Two rowsets with empty space between them.
+ * [ --- ] [ --- ]
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" }, { "C", "D" } }));
+
+ /* Three rowsets with empty space between them.
+ * [ --- ] [ --- ] [ --- ]
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" },
+ { "C", "D" },
+ { "E", "F" } }));
+
+ /* Three rowsets with empty space between them, and one is a single key.
+ * [ --- ] | [ --- ]
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" },
+ { "C", "C" },
+ { "D", "D" } }));
+
+ /* Two rowsets that completely overlap.
+ * [ --- ]
+ * [ --- ]
+ */
+ EXPECT_EQ(2.0, ComputeAverageRowsetHeight({ { "A", "B" }, { "A", "B" } }));
+
+ /* Three rowsets that completely overlap, but two are single keys, and the
+ * overlaps are on the boundaries.
+ * [ --- ]
+ * | |
+ */
+ EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" },
+ { "A", "A" },
+ { "B", "B" } }));
+
+ /* Three rowsets that completely overlap.
+ * [ --- ]
+ * [ --- ]
+ * [ --- ]
+ */
+ EXPECT_EQ(3.0, ComputeAverageRowsetHeight({ { "A", "B" },
+ { "A", "B" },
+ { "A", "B" } }));
+
+ /* Three rowsets that completely overlap, but one is a single key.
+ * [ --- ]
+ * [ --- ]
+ * |
+ */
+ EXPECT_EQ(2.0, ComputeAverageRowsetHeight({ { "A", "C" },
+ { "A", "C" },
+ { "B", "B" } }));
+
+ /* Two rowsets that partially overlap.
+ * [ --- ]
+ * [ --- ]
+ */
+ EXPECT_EQ(1.5, ComputeAverageRowsetHeight({ { "A", "C" }, { "B", "D" } }));
+
+ // Now the numbers stop being round so we'll check for being within some small
+ // range of an expected number.
+ constexpr auto epsilon = 0.001;
+
+ /* Three rowsets that partially overlap.
+ * [ --- ]
+ * [ --- ]
+ * [ --- ]
+ */
+ AssertWithinEpsilon(epsilon,
+ 1.66667,
+ ComputeAverageRowsetHeight({ { "A", "C" },
+ { "B", "D" },
+ { "C", "E" } }));
+
+ /* Three rowsets that partially overlap.
+ * [ --- ]
+ * [ --- ]
+ * [ --- ]
+ */
+ AssertWithinEpsilon(epsilon,
+ 2.33333,
+ ComputeAverageRowsetHeight({ { "A", "C" },
+ { "B", "D" },
+ { "A", "C" } }));
+
+ /* Five rowsets that partially overlap.
+ * [ --- ][ --- ]
+ * [ --- ]
+ * [ --- ][ --- ]
+ */
+ AssertWithinEpsilon(epsilon,
+ 2.6,
+ ComputeAverageRowsetHeight({ { "A", "C" }, { "C", "E" },
+ { "B", "D" },
+ { "A", "C" }, { "C", "E" } }));
+
+ /* A messy collection of rowsets overlapping in all kinds of ways.
+ * [ --- ][ --- ] [ --- ] [ --- ]
+ * [ --- ] [ --- ]
+ * [ --- ] | [ --- ] |
+ */
+ AssertWithinEpsilon(
+ epsilon,
+ 2.3125,
+ ComputeAverageRowsetHeight(
+ { { "A", "C" }, { "C", "E" }, { "F", "G" }, { "H", "J" },
+ { "B", "D" }, { "F", "I" },
+ { "A", "C" }, { "D", "D" }, { "F", "G" }, { "I", "I" } }));
+}
} // namespace tablet
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/compaction_policy.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/compaction_policy.cc b/src/kudu/tablet/compaction_policy.cc
index d1ead15..3048606 100644
--- a/src/kudu/tablet/compaction_policy.cc
+++ b/src/kudu/tablet/compaction_policy.cc
@@ -95,7 +95,7 @@ uint64_t BudgetedCompactionPolicy::target_rowset_size() const {
void BudgetedCompactionPolicy::SetupKnapsackInput(const RowSetTree &tree,
vector<RowSetInfo>* min_key,
vector<RowSetInfo>* max_key) {
- RowSetInfo::CollectOrdered(tree, min_key, max_key);
+ RowSetInfo::ComputeCdfAndCollectOrdered(tree, nullptr, min_key, max_key);
if (min_key->size() < 2) {
// require at least 2 rowsets to compact
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/rowset_info.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/rowset_info.cc b/src/kudu/tablet/rowset_info.cc
index 3227817..171a0e4 100644
--- a/src/kudu/tablet/rowset_info.cc
+++ b/src/kudu/tablet/rowset_info.cc
@@ -30,6 +30,7 @@
#include "kudu/gutil/casts.h"
#include "kudu/gutil/endian.h"
+#include "kudu/gutil/map-util.h"
#include "kudu/gutil/stringprintf.h"
#include "kudu/tablet/rowset.h"
#include "kudu/tablet/rowset_tree.h"
@@ -188,36 +189,40 @@ void RowSetInfo::Collect(const RowSetTree& tree, vector<RowSetInfo>* rsvec) {
}
}
-void RowSetInfo::CollectOrdered(const RowSetTree& tree,
- vector<RowSetInfo>* min_key,
- vector<RowSetInfo>* max_key) {
- // Resize
- size_t len = tree.all_rowsets().size();
- min_key->reserve(min_key->size() + len);
- max_key->reserve(max_key->size() + len);
+void RowSetInfo::ComputeCdfAndCollectOrdered(const RowSetTree& tree,
+ double* average_height,
+ vector<RowSetInfo>* info_by_min_key,
+ vector<RowSetInfo>* info_by_max_key) {
+ DCHECK((info_by_min_key && info_by_max_key) ||
+ (!info_by_min_key && !info_by_max_key))
+ << "'info_by_min_key' and 'info_by_max_key' must both be non-null or both be null";
// The collection process works as follows:
// For each sorted endpoint, first we identify whether it is a
// start or stop endpoint.
//
// At a start point, the associated rowset is added to the
- // "active" rowset mapping, allowing us to keep track of the index
- // of the rowset's RowSetInfo in the min_key vector.
+ // 'active' rowset mapping, allowing us to keep track of the index
+ // of the rowset's RowSetInfo in the 'info_by_min_key_tmp' vector.
//
- // At a stop point, the rowset is removed from the "active" map.
- // Note that the "active" map allows access to the incomplete
- // RowSetInfo that the RowSet maps to.
+ // At a stop point, the rowset is removed from the 'active' map.
+ // Note that the map allows access to the incomplete RowSetInfo that the
+ // RowSet maps to.
+ //
+ // The height of the tablet replica at the keys in between each successive
+ // pair of endpoints is active.size().
//
// The algorithm keeps track of its state - a "sliding window"
// across the keyspace - by maintaining the previous key and current
// value of the total width traversed over the intervals.
- Slice prev;
+ Slice prev_key;
unordered_map<RowSet*, RowSetInfo*> active;
- double total_width = 0.0f;
+ double total_width = 0.0;
+ double weighted_height_sum = 0.0;
- // We need to filter out the rowsets that aren't available before we process the endpoints,
- // else there's a race since we see endpoints twice and a delta compaction might finish in
- // between.
+ // We need to filter out the rowsets that aren't available before we process
+ // the endpoints, else there's a race since we see endpoints twice and a delta
+ // compaction might finish in between.
RowSetVector available_rowsets;
for (const auto& rs : tree.all_rowsets()) {
if (rs->IsAvailableForCompaction()) {
@@ -225,50 +230,74 @@ void RowSetInfo::CollectOrdered(const RowSetTree& tree,
}
}
+ size_t len = available_rowsets.size();
+ vector<RowSetInfo> info_by_min_key_tmp;
+ vector<RowSetInfo> info_by_max_key_tmp;
+
+ // NB: Since the algorithm will store pointers to elements in these vectors
+ // while they grow, the reserve calls are necessary for correctness.
+ info_by_min_key_tmp.reserve(len);
+ info_by_max_key_tmp.reserve(len);
+
+
RowSetTree available_rs_tree;
available_rs_tree.Reset(available_rowsets);
- for (const RowSetTree::RSEndpoint& rse :
- available_rs_tree.key_endpoints()) {
+ for (const auto& rse : available_rs_tree.key_endpoints()) {
RowSet* rs = rse.rowset_;
- const Slice& next = rse.slice_;
- double interval_width = WidthByDataSize(prev, next, active);
+ const Slice& next_key = rse.slice_;
+ double interval_width = WidthByDataSize(prev_key, next_key, active);
- // Increment active rowsets in min_key by the interval_width.
+ // For each active rowset, update the cdf value at the max key and the
+ // running total of weighted heights. They will be divided by the
+ // appropriate denominators at the end.
for (const auto& rs_rsi : active) {
RowSetInfo& cdf_rs = *rs_rsi.second;
cdf_rs.cdf_max_key_ += interval_width;
}
+ weighted_height_sum += active.size() * interval_width;
// Move sliding window
total_width += interval_width;
- prev = next;
+ prev_key = next_key;
// Add/remove current RowSetInfo
if (rse.endpoint_ == RowSetTree::START) {
- min_key->push_back(RowSetInfo(rs, total_width));
+ info_by_min_key_tmp.push_back(RowSetInfo(rs, total_width));
// Store reference from vector. This is safe b/c of reserve() above.
- active.insert(std::make_pair(rs, &min_key->back()));
+ EmplaceOrDie(&active, rs, &info_by_min_key_tmp.back());
} else if (rse.endpoint_ == RowSetTree::STOP) {
- // If not in active set, then STOP before START in endpoint tree
- RowSetInfo* cdf_rs = CHECK_NOTNULL(active[rs]);
+ // If the current rowset is not in the active set, then the rowset tree
+ // is inconsistent: an interval STOPs before it STARTs.
+ RowSetInfo* cdf_rs = FindOrDie(active, rs);
CHECK_EQ(cdf_rs->rowset(), rs) << "Inconsistent key interval tree.";
- CHECK_EQ(active.erase(rs), 1);
- max_key->push_back(*cdf_rs);
+ CHECK_NOTNULL(EraseKeyReturnValuePtr(&active, rs));
+ info_by_max_key_tmp.push_back(*cdf_rs);
} else {
LOG(FATAL) << "Undefined RowSet endpoint type.\n"
<< "\tExpected either RowSetTree::START=" << RowSetTree::START
<< " or RowSetTree::STOP=" << RowSetTree::STOP << ".\n"
<< "\tRecieved:\n"
<< "\t\tRowSet=" << rs->ToString() << "\n"
- << "\t\tKey=" << KUDU_REDACT(next.ToDebugString()) << "\n"
+ << "\t\tKey=" << KUDU_REDACT(next_key.ToDebugString()) << "\n"
<< "\t\tEndpointType=" << rse.endpoint_;
}
}
- CheckCollectOrderedCorrectness(*min_key, *max_key, total_width);
+ CheckCollectOrderedCorrectness(info_by_min_key_tmp,
+ info_by_max_key_tmp,
+ total_width);
+ FinalizeCDFVector(total_width, &info_by_min_key_tmp);
+ FinalizeCDFVector(total_width, &info_by_max_key_tmp);
- FinalizeCDFVector(min_key, total_width);
- FinalizeCDFVector(max_key, total_width);
+ if (average_height) {
+ *average_height = total_width > 0 ? weighted_height_sum / total_width
+ : 0.0;
+ }
+
+ if (info_by_min_key && info_by_max_key) {
+ *info_by_min_key = std::move(info_by_min_key_tmp);
+ *info_by_max_key = std::move(info_by_max_key_tmp);
+ }
}
void RowSetInfo::SplitKeyRange(const RowSetTree& tree,
@@ -370,8 +399,7 @@ uint64_t RowSetInfo::size_bytes(const ColumnId& col_id) const {
return extra_->rowset->OnDiskBaseDataColumnSize(col_id);
}
-void RowSetInfo::FinalizeCDFVector(vector<RowSetInfo>* vec,
- double quot) {
+void RowSetInfo::FinalizeCDFVector(double quot, vector<RowSetInfo>* vec) {
if (quot == 0) return;
for (RowSetInfo& cdf_rs : *vec) {
CHECK_GT(cdf_rs.size_mb_, 0) << "Expected file size to be at least 1MB "
@@ -380,8 +408,7 @@ void RowSetInfo::FinalizeCDFVector(vector<RowSetInfo>* vec,
<< " bytes.";
cdf_rs.cdf_min_key_ /= quot;
cdf_rs.cdf_max_key_ /= quot;
- cdf_rs.density_ = (cdf_rs.cdf_max_key() - cdf_rs.cdf_min_key())
- / cdf_rs.size_mb_;
+ cdf_rs.density_ = (cdf_rs.cdf_max_key() - cdf_rs.cdf_min_key()) / cdf_rs.size_mb_;
}
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/rowset_info.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/rowset_info.h b/src/kudu/tablet/rowset_info.h
index 0186837..437e357 100644
--- a/src/kudu/tablet/rowset_info.h
+++ b/src/kudu/tablet/rowset_info.h
@@ -44,11 +44,17 @@ class RowSetInfo {
// Appends the rowsets in no order without the cdf values set.
static void Collect(const RowSetTree& tree, std::vector<RowSetInfo>* rsvec);
- // Appends the rowsets in min-key and max-key sorted order, with
- // cdf values set.
- static void CollectOrdered(const RowSetTree& tree,
- std::vector<RowSetInfo>* min_key,
- std::vector<RowSetInfo>* max_key);
+
+ // From the rowset tree 'tree', computes the keyspace cdf and collects rowset
+ // information in min-key- and max-key-sorted order into 'info_by_min_key'
+ // and 'info_by_max_key', respectively. The average value of the height of the
+ // rowset tree is set into 'average_height', if it is not nullptr.
+ // If one of 'info_by_min_key' and 'info_by_max_key' is nullptr, the other
+ // must be.
+ static void ComputeCdfAndCollectOrdered(const RowSetTree& tree,
+ double* average_height,
+ std::vector<RowSetInfo>* info_by_min_key,
+ std::vector<RowSetInfo>* info_by_max_key);
// Split [start_key, stop_key) into primary key ranges by chunk size.
//
@@ -100,8 +106,7 @@ class RowSetInfo {
private:
explicit RowSetInfo(RowSet* rs, double init_cdf);
- static void FinalizeCDFVector(std::vector<RowSetInfo>* vec,
- double quot);
+ static void FinalizeCDFVector(double quot, std::vector<RowSetInfo>* vec);
// The size in MB, already clamped so that all rowsets have size at least
// 1MB. This is cached to avoid the branch during the selection hot path.
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/tablet.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 0a7a59a..fe4bd63 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -290,7 +290,17 @@ Status Tablet::Open() {
shared_ptr<RowSetTree> new_rowset_tree(new RowSetTree());
CHECK_OK(new_rowset_tree->Reset(rowsets_opened));
- // now that the current state is loaded, create the new MemRowSet with the next id
+ if (metrics_) {
+ // Compute the initial average height of the rowset tree.
+ double avg_height;
+ RowSetInfo::ComputeCdfAndCollectOrdered(*new_rowset_tree,
+ &avg_height,
+ nullptr,
+ nullptr);
+ metrics_->average_diskrowset_height->set_value(avg_height);
+ }
+
+ // Now that the current state is loaded, create the new MemRowSet with the next id.
shared_ptr<MemRowSet> new_mrs;
RETURN_NOT_OK(MemRowSet::Create(next_mrs_id_++, *schema(),
log_anchor_registry_.get(),
@@ -1037,6 +1047,7 @@ void Tablet::ModifyRowSetTree(const RowSetTree& old_tree,
rowsets_to_add.end(),
std::back_inserter(post_swap));
+
CHECK_OK(new_tree->Reset(post_swap));
}
@@ -1055,6 +1066,19 @@ void Tablet::AtomicSwapRowSetsUnlocked(const RowSetVector &to_remove,
to_remove, to_add, new_tree.get());
components_ = new TabletComponents(components_->memrowset, new_tree);
+
+ // Recompute the average rowset height.
+ // TODO(wdberkeley): We should be able to cache the computation of the CDF
+ // and average height and efficiently recompute it instead of doing it from
+ // scratch.
+ if (metrics_) {
+ double avg_height;
+ RowSetInfo::ComputeCdfAndCollectOrdered(*new_tree,
+ &avg_height,
+ nullptr,
+ nullptr);
+ metrics_->average_diskrowset_height->set_value(avg_height);
+ }
}
Status Tablet::DoMajorDeltaCompaction(const vector<ColumnId>& col_ids,
@@ -2296,8 +2320,9 @@ void Tablet::PrintRSLayout(ostream* o) {
*o << "</p>";
}
+ double avg_height;
vector<RowSetInfo> min, max;
- RowSetInfo::CollectOrdered(*rowsets_copy, &min, &max);
+ RowSetInfo::ComputeCdfAndCollectOrdered(*rowsets_copy, &avg_height, &min, &max);
DumpCompactionSVG(min, picked, o, /*print_xml_header=*/false);
// Compaction policy ignores rowsets unavailable for compaction. This is good,
@@ -2351,13 +2376,15 @@ void Tablet::PrintRSLayout(ostream* o) {
" <tr><td>Median</td><td>$3</td></tr>"
" <tr><td>Third quartile</td><td>$4</td></tr>"
" <tr><td>Max</td><td>$5</td></tr>"
+ " <tr><td>Avg. Height</td><td>$6</td></tr>"
"<tbody>",
num_rowsets,
HumanReadableNumBytes::ToString(size_bytes_min),
HumanReadableNumBytes::ToString(size_bytes_first_quartile),
HumanReadableNumBytes::ToString(size_bytes_median),
HumanReadableNumBytes::ToString(size_bytes_third_quartile),
- HumanReadableNumBytes::ToString(size_bytes_max));
+ HumanReadableNumBytes::ToString(size_bytes_max),
+ avg_height);
*o << "</table>" << endl;
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/tablet_metrics.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet_metrics.cc b/src/kudu/tablet/tablet_metrics.cc
index a3b6b41..5aa4fa2 100644
--- a/src/kudu/tablet/tablet_metrics.cc
+++ b/src/kudu/tablet/tablet_metrics.cc
@@ -255,10 +255,15 @@ METRIC_DEFINE_counter(tablet, leader_memory_pressure_rejections,
kudu::MetricUnit::kRequests,
"Number of RPC requests rejected due to memory pressure while LEADER.");
+METRIC_DEFINE_gauge_double(tablet, average_diskrowset_height, "Average DiskRowSet Height",
+ kudu::MetricUnit::kUnits,
+ "Average height of the diskrowsets in this tablet "
+ "replica. The larger the average height, the more "
+ "uncompacted the tablet replica is.");
+
using strings::Substitute;
using std::unordered_map;
-
namespace kudu {
namespace tablet {
@@ -307,7 +312,8 @@ TabletMetrics::TabletMetrics(const scoped_refptr<MetricEntity>& entity)
MINIT(undo_delta_block_gc_init_duration),
MINIT(undo_delta_block_gc_delete_duration),
MINIT(undo_delta_block_gc_perform_duration),
- MINIT(leader_memory_pressure_rejections) {
+ MINIT(leader_memory_pressure_rejections),
+ GINIT(average_diskrowset_height) {
}
#undef MINIT
#undef GINIT
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/tablet_metrics.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet_metrics.h b/src/kudu/tablet/tablet_metrics.h
index 9fe87b7..c60b9aa 100644
--- a/src/kudu/tablet/tablet_metrics.h
+++ b/src/kudu/tablet/tablet_metrics.h
@@ -43,7 +43,7 @@ struct TabletMetrics {
// This allocates temporary scratch space from work_arena.
void AddProbeStats(const ProbeStats* stats_array, int len, Arena* work_arena);
- // Operation rates
+ // Operation rates.
scoped_refptr<Counter> rows_inserted;
scoped_refptr<Counter> rows_upserted;
scoped_refptr<Counter> rows_updated;
@@ -51,7 +51,7 @@ struct TabletMetrics {
scoped_refptr<Counter> insertions_failed_dup_key;
scoped_refptr<Counter> upserts_as_updates;
- // Scanner metrics
+ // Scanner metrics.
scoped_refptr<Counter> scanner_rows_returned;
scoped_refptr<Counter> scanner_cells_returned;
scoped_refptr<Counter> scanner_bytes_returned;
@@ -61,7 +61,7 @@ struct TabletMetrics {
scoped_refptr<Counter> scans_started;
scoped_refptr<AtomicGauge<size_t>> tablet_active_scanners;
- // Probe stats
+ // Probe stats.
scoped_refptr<Counter> bloom_lookups;
scoped_refptr<Counter> key_file_lookups;
scoped_refptr<Counter> delta_file_lookups;
@@ -98,6 +98,9 @@ struct TabletMetrics {
scoped_refptr<Histogram> undo_delta_block_gc_perform_duration;
scoped_refptr<Counter> leader_memory_pressure_rejections;
+
+ // Compaction metrics.
+ scoped_refptr<AtomicGauge<double>> average_diskrowset_height;
};
} // namespace tablet
[2/2] kudu git commit: Update Java README Intellij test runner
instructions
Posted by da...@apache.org.
Update Java README Intellij test runner instructions
Change-Id: Ie372417350f50a7d4f5567cd991ac5d886329cdb
Reviewed-on: http://gerrit.cloudera.org:8080/11700
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Grant Henke <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/09848055
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/09848055
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/09848055
Branch: refs/heads/master
Commit: 09848055b089be98754b82f6cae74ddc777778f6
Parents: d3684a7
Author: Dan Burkert <da...@apache.org>
Authored: Tue Oct 16 13:31:40 2018 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Tue Oct 16 20:58:27 2018 +0000
----------------------------------------------------------------------
java/README.adoc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/09848055/java/README.adoc
----------------------------------------------------------------------
diff --git a/java/README.adoc b/java/README.adoc
index 5049b51..caccea8 100644
--- a/java/README.adoc
+++ b/java/README.adoc
@@ -140,7 +140,8 @@ to import the Gradle project.
For the most consistent behavior on the command line and
in the IDE, be sure to enable `Delegate IDE build/run actions to gradle`
-as described https://www.jetbrains.com/help/idea/gradle.html#delegate_build_gradle[here].
+and run tests using the `Gradle Test Runner` as described
+https://www.jetbrains.com/help/idea/gradle.html#delegate_build_gradle[here].
=== Eclipse