You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2017/12/22 19:41:51 UTC
kudu git commit: Improve scans dashboard
Repository: kudu
Updated Branches:
refs/heads/master d78b2727d -> 5356334ef
Improve scans dashboard
* Adds a ring buffer to the scanners manager which holds historical scan
details, so that the scan dashboard can show recently completed scans.
The size of the buffer is configurable with a new experimental flag,
--scan-history-count which defaults to 20. A new 'state' column has
been added to indicate the state of the scan (active, expired, failed,
or complete).
* Mustache-ifies the scans dashboard.
* The dashboard previously showed raw values, with pretty-printed values
as a tooltip. In order to match the maintenance manager dashboard,
these have been swapped so that pretty-printed values are more
prominent.
* Adds a new pseudo-SQL query description column, replacing the
predicates columns. This allows the table, projected columns, and
optimized predicates to be shown in a very concise way. Previously the
table name and projected columns weren't exposed at all.
* Removes 'from disk' from statistics names, since it was misleading, or
downright wrong. The tooltip for the cells, bytes, and cblocks read
columns now indicates that the values could have been read from cache or
from disk. It would probably be nice to add a cache hit rate to the
stats, but I didn't want to include it in this commit as it's already
big.
* Adds aggregated totals for scan stats.
* A bunch of other polish, like tooltip descriptions of the
non-completely-obvious column headers, and adding a link from the
tablet ID to the replica's page.
example: https://i.imgur.com/WLMK263.png
Change-Id: Iadbdee00d8a343fd3728c6ec8ee252d64d0e416a
Reviewed-on: http://gerrit.cloudera.org:8080/8891
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5356334e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5356334e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5356334e
Branch: refs/heads/master
Commit: 5356334ef3b4949c9dba7bc6a9f7f9d5262c4281
Parents: d78b272
Author: Dan Burkert <da...@apache.org>
Authored: Fri Dec 15 10:45:42 2017 -0800
Committer: Dan Burkert <da...@apache.org>
Committed: Fri Dec 22 19:41:34 2017 +0000
----------------------------------------------------------------------
src/kudu/cfile/cfile_reader.cc | 6 +-
src/kudu/common/column_predicate-test.cc | 2 +-
src/kudu/common/column_predicate.cc | 43 ++---
src/kudu/common/encoded_key.cc | 22 ++-
src/kudu/common/generic_iterators.cc | 2 +-
src/kudu/common/iterator_stats.cc | 50 ++---
src/kudu/common/iterator_stats.h | 19 +-
src/kudu/common/scan_spec-test.cc | 84 ++++-----
.../integration-tests/linked_list-test-util.h | 1 +
src/kudu/tablet/cfile_set-test.cc | 15 +-
src/kudu/tablet/tablet-pushdown-test.cc | 13 +-
src/kudu/tserver/scanners.cc | 149 ++++++++++++++-
src/kudu/tserver/scanners.h | 57 +++++-
src/kudu/tserver/tablet_service.cc | 21 +--
src/kudu/tserver/tserver_path_handlers.cc | 187 +++++++++----------
src/kudu/tserver/tserver_path_handlers.h | 10 +-
www/scans.mustache | 72 +++++++
17 files changed, 505 insertions(+), 248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/cfile/cfile_reader.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile_reader.cc b/src/kudu/cfile/cfile_reader.cc
index 6f19487..b3ffad6 100644
--- a/src/kudu/cfile/cfile_reader.cc
+++ b/src/kudu/cfile/cfile_reader.cc
@@ -939,9 +939,9 @@ Status CFileIterator::ReadCurrentDataBlock(const IndexTreeIterator &idx_iter,
num_rows_in_block = bd->Count();
}
- io_stats_.cells_read_from_disk += num_rows_in_block;
- io_stats_.data_blocks_read_from_disk++;
- io_stats_.bytes_read_from_disk += data_block.size();
+ io_stats_.cells_read += num_rows_in_block;
+ io_stats_.cblocks_read++;
+ io_stats_.bytes_read += data_block.size();
prep_block->idx_in_block_ = 0;
prep_block->num_rows_in_block_ = num_rows_in_block;
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/column_predicate-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/column_predicate-test.cc b/src/kudu/common/column_predicate-test.cc
index 93a4e12..7548e9c 100644
--- a/src/kudu/common/column_predicate-test.cc
+++ b/src/kudu/common/column_predicate-test.cc
@@ -1096,7 +1096,7 @@ TEST_F(TestColumnPredicate, TestRedaction) {
ASSERT_NE("", gflags::SetCommandLineOption("redact", "log"));
ColumnSchema column_i32("a", INT32, true);
int32_t one_32 = 1;
- ASSERT_EQ("`a` = <redacted>", ColumnPredicate::Equality(column_i32, &one_32).ToString());
+ ASSERT_EQ("a = <redacted>", ColumnPredicate::Equality(column_i32, &one_32).ToString());
}
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/column_predicate.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/column_predicate.cc b/src/kudu/common/column_predicate.cc
index 1197b9f..8a1dea3 100644
--- a/src/kudu/common/column_predicate.cc
+++ b/src/kudu/common/column_predicate.cc
@@ -27,8 +27,10 @@
#include "kudu/common/rowblock.h"
#include "kudu/common/schema.h"
#include "kudu/common/types.h"
+#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/util/bitmap.h"
+#include "kudu/util/logging.h"
#include "kudu/util/memory/arena.h"
using std::move;
@@ -627,41 +629,36 @@ void ColumnPredicate::Evaluate(const ColumnBlock& block, SelectionVector* sel) c
string ColumnPredicate::ToString() const {
switch (predicate_type()) {
- case PredicateType::None: return strings::Substitute("`$0` NONE", column_.name());
+ case PredicateType::None: return strings::Substitute("$0 NONE", column_.name());
case PredicateType::Range: {
if (lower_ == nullptr) {
- return strings::Substitute("`$0` < $1", column_.name(), column_.Stringify(upper_));
- } else if (upper_ == nullptr) {
- return strings::Substitute("`$0` >= $1", column_.name(), column_.Stringify(lower_));
- } else {
- return strings::Substitute("`$0` >= $1 AND `$0` < $2",
- column_.name(),
- column_.Stringify(lower_),
- column_.Stringify(upper_));
+ return strings::Substitute("$0 < $1", column_.name(), column_.Stringify(upper_));
}
+ if (upper_ == nullptr) {
+ return strings::Substitute("$0 >= $1", column_.name(), column_.Stringify(lower_));
+ }
+ return strings::Substitute("$0 >= $1 AND $0 < $2",
+ column_.name(),
+ column_.Stringify(lower_),
+ column_.Stringify(upper_));
+
};
case PredicateType::Equality: {
- return strings::Substitute("`$0` = $1", column_.name(), column_.Stringify(lower_));
+ return strings::Substitute("$0 = $1", column_.name(), column_.Stringify(lower_));
};
case PredicateType::IsNotNull: {
- return strings::Substitute("`$0` IS NOT NULL", column_.name());
+ return strings::Substitute("$0 IS NOT NULL", column_.name());
};
case PredicateType::IsNull: {
- return strings::Substitute("`$0` IS NULL", column_.name());
+ return strings::Substitute("$0 IS NULL", column_.name());
};
case PredicateType::InList: {
- string ss = "`";
+ string ss;
ss.append(column_.name());
- ss.append("` IN (");
- bool is_first = true;
- for (auto* value : values_) {
- if (is_first) {
- is_first = false;
- } else {
- ss.append(", ");
- }
- ss.append(column_.Stringify(value));
- }
+ ss.append(" IN (");
+ ss.append(KUDU_REDACT(JoinMapped(values_,
+ [&] (const void* value) { return column_.Stringify(value); },
+ ", ")));
ss.append(")");
return ss;
};
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/encoded_key.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/encoded_key.cc b/src/kudu/common/encoded_key.cc
index 069e04a..b844dd9 100644
--- a/src/kudu/common/encoded_key.cc
+++ b/src/kudu/common/encoded_key.cc
@@ -119,21 +119,23 @@ Status EncodedKey::IncrementEncodedKey(const Schema& tablet_schema,
}
string EncodedKey::Stringify(const Schema &schema) const {
- if (num_key_cols_ == 1) {
- return schema.column(0).Stringify(raw_keys_.front());
- }
+ DCHECK_EQ(schema.num_key_columns(), num_key_cols_);
+ DCHECK_EQ(schema.num_key_columns(), raw_keys_.size());
faststring s;
s.append("(");
- for (int i = 0; i < num_key_cols_; i++) {
+ for (int i = 0; i < raw_keys_.size(); i++) {
if (i > 0) {
- s.append(",");
- }
- if (i < raw_keys_.size()) {
- s.append(schema.column(i).Stringify(raw_keys_[i]));
- } else {
- s.append("*");
+ if (schema.column(i).type_info()->IsMinValue(raw_keys_[i])) {
+ // If the value is the minimum, short-circuit to avoid printing keys such as
+ // '(2, -9223372036854775808, -9223372036854775808, -9223372036854775808)',
+ // and instead print '(2)'. The minimum values are usually filled in
+ // automatically upon decoding, so it makes sense to omit them.
+ break;
+ }
+ s.append(", ");
}
+ s.append(schema.column(i).Stringify(raw_keys_[i]));
}
s.append(")");
return s.ToString();
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/generic_iterators.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/generic_iterators.cc b/src/kudu/common/generic_iterators.cc
index 66b5d8a..76971ac 100644
--- a/src/kudu/common/generic_iterators.cc
+++ b/src/kudu/common/generic_iterators.cc
@@ -71,7 +71,7 @@ void AddIterStats(const RowwiseIterator& iter,
iter.GetIteratorStats(&iter_stats);
DCHECK_EQ(stats->size(), iter_stats.size());
for (int i = 0; i < iter_stats.size(); i++) {
- (*stats)[i].AddStats(iter_stats[i]);
+ (*stats)[i] += iter_stats[i];
}
}
} // anonymous namespace
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/iterator_stats.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/iterator_stats.cc b/src/kudu/common/iterator_stats.cc
index 77eee10..42d2b57 100644
--- a/src/kudu/common/iterator_stats.cc
+++ b/src/kudu/common/iterator_stats.cc
@@ -27,39 +27,47 @@ using std::string;
using strings::Substitute;
IteratorStats::IteratorStats()
- : data_blocks_read_from_disk(0),
- bytes_read_from_disk(0),
- cells_read_from_disk(0) {
+ : cells_read(0),
+ bytes_read(0),
+ cblocks_read(0) {
}
string IteratorStats::ToString() const {
- return Substitute("data_blocks_read_from_disk=$0 "
- "bytes_read_from_disk=$1 "
- "cells_read_from_disk=$2",
- data_blocks_read_from_disk,
- bytes_read_from_disk,
- cells_read_from_disk);
+ return Substitute("cells_read=$0 bytes_read=$1 cblocks_read=$2",
+ cells_read, bytes_read, cblocks_read);
}
-void IteratorStats::AddStats(const IteratorStats& other) {
- data_blocks_read_from_disk += other.data_blocks_read_from_disk;
- bytes_read_from_disk += other.bytes_read_from_disk;
- cells_read_from_disk += other.cells_read_from_disk;
+IteratorStats& IteratorStats::operator+=(const IteratorStats& other) {
+ cells_read += other.cells_read;
+ bytes_read += other.bytes_read;
+ cblocks_read += other.cblocks_read;
DCheckNonNegative();
+ return *this;
}
-void IteratorStats::SubtractStats(const IteratorStats& other) {
- data_blocks_read_from_disk -= other.data_blocks_read_from_disk;
- bytes_read_from_disk -= other.bytes_read_from_disk;
- cells_read_from_disk -= other.cells_read_from_disk;
+IteratorStats& IteratorStats::operator-=(const IteratorStats& other) {
+ cells_read -= other.cells_read;
+ bytes_read -= other.bytes_read;
+ cblocks_read -= other.cblocks_read;
DCheckNonNegative();
+ return *this;
}
-void IteratorStats::DCheckNonNegative() const {
- DCHECK_GE(data_blocks_read_from_disk, 0);
- DCHECK_GE(bytes_read_from_disk, 0);
- DCHECK_GE(cells_read_from_disk, 0);
+IteratorStats IteratorStats::operator+(const IteratorStats& other) {
+ IteratorStats copy = *this;
+ copy += other;
+ return copy;
}
+IteratorStats IteratorStats::operator-(const IteratorStats& other) {
+ IteratorStats copy = *this;
+ copy -= other;
+ return copy;
+}
+void IteratorStats::DCheckNonNegative() const {
+ DCHECK_GE(cells_read, 0);
+ DCHECK_GE(bytes_read, 0);
+ DCHECK_GE(cblocks_read, 0);
+}
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/iterator_stats.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/iterator_stats.h b/src/kudu/common/iterator_stats.h
index ed2bcb9..542c87f 100644
--- a/src/kudu/common/iterator_stats.h
+++ b/src/kudu/common/iterator_stats.h
@@ -27,23 +27,24 @@ struct IteratorStats {
std::string ToString() const;
- // The number of data blocks read from disk (or cache) by the iterator.
- int64_t data_blocks_read_from_disk;
+ // The number of cells which were read from disk -- regardless of whether
+ // they were decoded/materialized.
+ int64_t cells_read;
// The number of bytes read from disk (or cache) by the iterator.
- int64_t bytes_read_from_disk;
+ int64_t bytes_read;
- // The number of cells which were read from disk -- regardless of whether
- // they were decoded/materialized.
- int64_t cells_read_from_disk;
+ // The number of CFile data blocks read from disk (or cache) by the iterator.
+ int64_t cblocks_read;
// Add statistics contained 'other' to this object (for each field
// in this object, increment it by the value of the equivalent field
// in 'other').
- void AddStats(const IteratorStats& other);
+ IteratorStats& operator+=(const IteratorStats& other);
+ IteratorStats& operator-=(const IteratorStats& other);
- // Same, except subtract.
- void SubtractStats(const IteratorStats& other);
+ IteratorStats operator+(const IteratorStats& other);
+ IteratorStats operator-(const IteratorStats& other);
private:
// DCHECK that all of the stats are non-negative. This is a no-op in
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/common/scan_spec-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/scan_spec-test.cc b/src/kudu/common/scan_spec-test.cc
index 9547b30..e24f9d2 100644
--- a/src/kudu/common/scan_spec-test.cc
+++ b/src/kudu/common/scan_spec-test.cc
@@ -153,9 +153,9 @@ TEST_F(CompositeIntKeysTest, TestSimplify) {
SCOPED_TRACE(spec.ToString(schema_));
ASSERT_EQ(3, spec.predicates().size());
- ASSERT_EQ("`a` = 127", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` >= 3 AND `b` < 101", FindOrDie(spec.predicates(), "b").ToString());
- ASSERT_EQ("`c` < 65", FindOrDie(spec.predicates(), "c").ToString());
+ ASSERT_EQ("a = 127", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b >= 3 AND b < 101", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("c < 65", FindOrDie(spec.predicates(), "c").ToString());
}
// Predicate: a == 64
@@ -197,7 +197,7 @@ TEST_F(CompositeIntKeysTest, TestConsecutiveLowerRangePredicates) {
AddPredicate<int8_t>(&spec, "c", GE, 5);
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
- EXPECT_EQ("PK >= (int8 a=3, int8 b=4, int8 c=5) AND `b` >= 4 AND `c` >= 5",
+ EXPECT_EQ("PK >= (int8 a=3, int8 b=4, int8 c=5) AND b >= 4 AND c >= 5",
spec.ToString(schema_));
}
@@ -209,7 +209,7 @@ TEST_F(CompositeIntKeysTest, TestConsecutiveUpperRangePredicates) {
AddPredicate<int8_t>(&spec, "c", LE, 5);
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
- EXPECT_EQ("PK < (int8 a=3, int8 b=4, int8 c=6) AND `b` < 5 AND `c` < 6",
+ EXPECT_EQ("PK < (int8 a=3, int8 b=4, int8 c=6) AND b < 5 AND c < 6",
spec.ToString(schema_));
}
@@ -223,7 +223,7 @@ TEST_F(CompositeIntKeysTest, TestEqualityAndConsecutiveLowerRangePredicates) {
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ("PK >= (int8 a=3, int8 b=4, int8 c=5) AND "
"PK < (int8 a=4, int8 b=-128, int8 c=-128) AND "
- "`c` >= 5", spec.ToString(schema_));
+ "c >= 5", spec.ToString(schema_));
}
// Predicates: a = 3 AND 4 <= b <= 14 AND 15 <= c <= 15
@@ -238,7 +238,7 @@ TEST_F(CompositeIntKeysTest, TestEqualityAndConsecutiveRangePredicates) {
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ("PK >= (int8 a=3, int8 b=4, int8 c=5) AND "
"PK < (int8 a=3, int8 b=14, int8 c=16) AND "
- "`c` >= 5 AND `c` < 16", spec.ToString(schema_));
+ "c >= 5 AND c < 16", spec.ToString(schema_));
}
// Test a predicate on a non-prefix part of the key. Can't be pushed.
@@ -249,8 +249,8 @@ TEST_F(CompositeIntKeysTest, TestNonPrefix) {
AddPredicate<int8_t>(&spec, "b", EQ, 64);
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
- // Expect: nothing pushed (predicate is still on `b`, not PK)
- EXPECT_EQ("`b` = 64", spec.ToString(schema_));
+ // Expect: nothing pushed (predicate is still on b, not PK)
+ EXPECT_EQ("b = 64", spec.ToString(schema_));
}
// Test what happens when an upper bound on a cell is equal to the maximum
@@ -308,7 +308,7 @@ TEST_F(CompositeIntKeysTest, TestNoErasePredicates) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
EXPECT_EQ("PK >= (int8 a=126, int8 b=-128, int8 c=-128) AND "
"PK < (int8 a=127, int8 b=-128, int8 c=-128) AND "
- "`a` = 126", spec.ToString(schema_));
+ "a = 126", spec.ToString(schema_));
}
// Test that, if pushed predicates are erased, that we don't
@@ -326,7 +326,7 @@ TEST_F(CompositeIntKeysTest, TestNoErasePredicates2) {
// The predicate on column A should be pushed while "c" remains.
EXPECT_EQ("PK >= (int8 a=126, int8 b=-128, int8 c=-128) AND "
"PK < (int8 a=127, int8 b=-128, int8 c=-128) AND "
- "`c` = 126", spec.ToString(schema_));
+ "c = 126", spec.ToString(schema_));
}
// Test that predicates added out of key order are OK.
@@ -352,7 +352,7 @@ TEST_F(CompositeIntKeysTest, TestIsNotNullPushdown) {
spec.AddPredicate(ColumnPredicate::IsNotNull(schema_.column(3)));
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
- EXPECT_EQ("`d` IS NOT NULL", spec.ToString(schema_));
+ EXPECT_EQ("d IS NOT NULL", spec.ToString(schema_));
}
// Test that IN list predicates get pushed into the primary key bounds.
@@ -364,7 +364,7 @@ TEST_F(CompositeIntKeysTest, TestInListPushdown) {
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ("PK >= (int8 a=0, int8 b=50, int8 c=-128) AND "
"PK < (int8 a=10, int8 b=101, int8 c=-128) AND "
- "`a` IN (0, 10) AND `b` IN (50, 100)",
+ "a IN (0, 10) AND b IN (50, 100)",
spec.ToString(schema_));
}
@@ -379,14 +379,14 @@ TEST_F(CompositeIntKeysTest, TestInListPushdownWithRange) {
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ("PK >= (int8 a=10, int8 b=50, int8 c=-128) AND "
"PK < (int8 a=100, int8 b=101, int8 c=-128) AND "
- "`b` IN (50, 100)",
+ "b IN (50, 100)",
spec.ToString(schema_));
// Test redaction.
ASSERT_NE("", gflags::SetCommandLineOption("redact", "log"));
EXPECT_EQ("PK >= (int8 a=<redacted>, int8 b=<redacted>, int8 c=<redacted>) AND "
"PK < (int8 a=<redacted>, int8 b=<redacted>, int8 c=<redacted>) AND "
- "`b` IN (<redacted>, <redacted>)",
+ "b IN (<redacted>)",
spec.ToString(schema_));
}
@@ -413,7 +413,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_LowerBound) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` >= 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a >= 10", FindOrDie(spec.predicates(), "a").ToString());
}
{ // key >= (10, 11, min)
ScanSpec spec;
@@ -427,7 +427,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_LowerBound) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` >= 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a >= 10", FindOrDie(spec.predicates(), "a").ToString());
}
{ // key >= (10, min, min)
ScanSpec spec;
@@ -441,7 +441,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_LowerBound) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` >= 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a >= 10", FindOrDie(spec.predicates(), "a").ToString());
}
}
@@ -461,7 +461,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_UpperBound) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` < 11", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a < 11", FindOrDie(spec.predicates(), "a").ToString());
}
{
// key < (10, 11, min)
@@ -476,7 +476,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_UpperBound) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` < 11", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a < 11", FindOrDie(spec.predicates(), "a").ToString());
}
{
// key < (10, min, min)
@@ -491,7 +491,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_UpperBound) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` < 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a < 10", FindOrDie(spec.predicates(), "a").ToString());
}
}
@@ -518,9 +518,9 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_BothBounds) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(3, spec.predicates().size());
- ASSERT_EQ("`a` = 10", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` = 11", FindOrDie(spec.predicates(), "b").ToString());
- ASSERT_EQ("`c` = 12", FindOrDie(spec.predicates(), "c").ToString());
+ ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b = 11", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("c = 12", FindOrDie(spec.predicates(), "c").ToString());
}
{
// key >= (10, 11, 12)
@@ -542,9 +542,9 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_BothBounds) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(3, spec.predicates().size());
- ASSERT_EQ("`a` = 10", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` = 11", FindOrDie(spec.predicates(), "b").ToString());
- ASSERT_EQ("`c` >= 12 AND `c` < 14", FindOrDie(spec.predicates(), "c").ToString());
+ ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b = 11", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("c >= 12 AND c < 14", FindOrDie(spec.predicates(), "c").ToString());
}
{
// key >= (10, 11, 12)
@@ -566,9 +566,9 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_BothBounds) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(3, spec.predicates().size());
- ASSERT_EQ("`a` = 10", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` = 11", FindOrDie(spec.predicates(), "b").ToString());
- ASSERT_EQ("`c` >= 12", FindOrDie(spec.predicates(), "c").ToString());
+ ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b = 11", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("c >= 12", FindOrDie(spec.predicates(), "c").ToString());
}
{
// key >= (10, 11, 12)
@@ -590,8 +590,8 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_BothBounds) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(2, spec.predicates().size());
- ASSERT_EQ("`a` = 10", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` >= 11 AND `b` < 13", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b >= 11 AND b < 13", FindOrDie(spec.predicates(), "b").ToString());
}
{
// key >= (10, 11, 12)
@@ -613,8 +613,8 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_BothBounds) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(2, spec.predicates().size());
- ASSERT_EQ("`a` = 10", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` >= 11", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b >= 11", FindOrDie(spec.predicates(), "b").ToString());
}
{
// key >= (10, min, min)
@@ -636,7 +636,7 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_BothBounds) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(1, spec.predicates().size());
- ASSERT_EQ("`a` >= 10 AND `a` < 12", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("a >= 10 AND a < 12", FindOrDie(spec.predicates(), "a").ToString());
}
}
@@ -668,9 +668,9 @@ TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_WithPredicates) {
spec.OptimizeScan(schema_, &arena_, &pool_, false);
ASSERT_EQ(3, spec.predicates().size());
- ASSERT_EQ("`a` = 10", FindOrDie(spec.predicates(), "a").ToString());
- ASSERT_EQ("`b` >= 15 AND `b` < 90", FindOrDie(spec.predicates(), "b").ToString());
- ASSERT_EQ("`c` >= 3 AND `c` < 101", FindOrDie(spec.predicates(), "c").ToString());
+ ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
+ ASSERT_EQ("b >= 15 AND b < 90", FindOrDie(spec.predicates(), "b").ToString());
+ ASSERT_EQ("c >= 3 AND c < 101", FindOrDie(spec.predicates(), "c").ToString());
}
// Tests for String parts in composite keys
@@ -719,7 +719,7 @@ TEST_F(CompositeIntStringKeysTest, TestDecreaseUpperBoundKey) {
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ(R"(PK < (int8 a=63, string b="abc", string c="") AND )"
- R"(`b` < "abc" AND `c` < "def\000")",
+ R"(b < "abc" AND c < "def\000")",
spec.ToString(schema_));
}
{
@@ -730,7 +730,7 @@ TEST_F(CompositeIntStringKeysTest, TestDecreaseUpperBoundKey) {
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ(R"(PK < (int8 a=63, string b="abc", string c="def") AND )"
- R"(`b` < "abc\000" AND `c` < "def")",
+ R"(b < "abc\000" AND c < "def")",
spec.ToString(schema_));
}
{
@@ -741,7 +741,7 @@ TEST_F(CompositeIntStringKeysTest, TestDecreaseUpperBoundKey) {
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ(R"(PK < (int8 a=63, string b="abc", string c="def\000") AND )"
- R"(`b` < "abc\000" AND `c` < "def\000")",
+ R"(b < "abc\000" AND c < "def\000")",
spec.ToString(schema_));
}
{
@@ -752,7 +752,7 @@ TEST_F(CompositeIntStringKeysTest, TestDecreaseUpperBoundKey) {
SCOPED_TRACE(spec.ToString(schema_));
spec.OptimizeScan(schema_, &arena_, &pool_, true);
EXPECT_EQ(R"(PK < (int8 a=63, string b="abc", string c="") AND )"
- R"(`b` < "abc" AND `c` < "def")",
+ R"(b < "abc" AND c < "def")",
spec.ToString(schema_));
}
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/integration-tests/linked_list-test-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/linked_list-test-util.h b/src/kudu/integration-tests/linked_list-test-util.h
index 39b9dcd..df698c8 100644
--- a/src/kudu/integration-tests/linked_list-test-util.h
+++ b/src/kudu/integration-tests/linked_list-test-util.h
@@ -325,6 +325,7 @@ class PeriodicWebUIChecker {
}
ts_pages.emplace_back("/maintenance-manager");
ts_pages.emplace_back("/mem-trackers");
+ ts_pages.emplace_back("/scans");
// Generate list of urls for each master and tablet server
for (int i = 0; i < cluster.num_masters(); i++) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tablet/cfile_set-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/cfile_set-test.cc b/src/kudu/tablet/cfile_set-test.cc
index 3f10929..11e0f8b 100644
--- a/src/kudu/tablet/cfile_set-test.cc
+++ b/src/kudu/tablet/cfile_set-test.cc
@@ -227,15 +227,15 @@ TEST_F(TestCFileSet, TestPartiallyMaterialize) {
}
// Since we pushed down the block size, we expect to have read 100+ blocks of column 0
- ASSERT_GT(stats[0].data_blocks_read_from_disk, 100);
+ ASSERT_GT(stats[0].cblocks_read, 100);
// Since we didn't ever materialize column 2, we shouldn't have read any data blocks.
- ASSERT_EQ(0, stats[2].data_blocks_read_from_disk);
+ ASSERT_EQ(0, stats[2].cblocks_read);
// Column 0 and 1 skipped a lot of blocks, so should not have read all of the cells
// from either column.
- ASSERT_LT(stats[0].cells_read_from_disk, kNumRows * 3 / 4);
- ASSERT_LT(stats[1].cells_read_from_disk, kNumRows * 3 / 4);
+ ASSERT_LT(stats[0].cells_read, kNumRows * 3 / 4);
+ ASSERT_LT(stats[1].cells_read, kNumRows * 3 / 4);
}
TEST_F(TestCFileSet, TestIteratePartialSchema) {
@@ -316,9 +316,10 @@ TEST_F(TestCFileSet, TestRangeScan) {
// Since it's a small range, it should be all in one data block in each column.
vector<IteratorStats> stats;
iter->GetIteratorStats(&stats);
- EXPECT_EQ(stats[0].data_blocks_read_from_disk, 1);
- EXPECT_EQ(stats[1].data_blocks_read_from_disk, 1);
- EXPECT_EQ(stats[2].data_blocks_read_from_disk, 1);
+ ASSERT_EQ(3, stats.size());
+ EXPECT_EQ(1, stats[0].cblocks_read);
+ EXPECT_EQ(1, stats[1].cblocks_read);
+ EXPECT_EQ(1, stats[2].cblocks_read);
}
// Several other black-box tests for range scans. These are similar to
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tablet/tablet-pushdown-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet-pushdown-test.cc b/src/kudu/tablet/tablet-pushdown-test.cc
index 3023697..2297afc 100644
--- a/src/kudu/tablet/tablet-pushdown-test.cc
+++ b/src/kudu/tablet/tablet-pushdown-test.cc
@@ -160,8 +160,8 @@ class TabletPushdownTest : public KuduTabletTest,
vector<IteratorStats> stats;
iter->GetIteratorStats(&stats);
for (const IteratorStats& col_stats : stats) {
- EXPECT_EQ(expected_blocks_from_disk, col_stats.data_blocks_read_from_disk);
- EXPECT_EQ(expected_rows_from_disk, col_stats.cells_read_from_disk);
+ EXPECT_EQ(expected_blocks_from_disk, col_stats.cblocks_read);
+ EXPECT_EQ(expected_rows_from_disk, col_stats.cells_read);
}
}
}
@@ -282,11 +282,12 @@ TEST_F(TabletSparsePushdownTest, Kudu2231) {
vector<IteratorStats> stats;
iter->GetIteratorStats(&stats);
- EXPECT_EQ(1, stats[0].data_blocks_read_from_disk);
- EXPECT_EQ(1, stats[1].data_blocks_read_from_disk);
+ ASSERT_EQ(2, stats.size());
+ EXPECT_EQ(1, stats[0].cblocks_read);
+ EXPECT_EQ(1, stats[1].cblocks_read);
- EXPECT_EQ(400, stats[0].cells_read_from_disk);
- EXPECT_EQ(400, stats[1].cells_read_from_disk);
+ EXPECT_EQ(400, stats[0].cells_read);
+ EXPECT_EQ(400, stats[1].cells_read);
}
} // namespace tablet
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tserver/scanners.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/scanners.cc b/src/kudu/tserver/scanners.cc
index d328ae8..eaf4836 100644
--- a/src/kudu/tserver/scanners.cc
+++ b/src/kudu/tserver/scanners.cc
@@ -16,12 +16,15 @@
// under the License.
#include "kudu/tserver/scanners.h"
+#include <algorithm>
#include <cstdint>
#include <mutex>
#include <ostream>
#include <gflags/gflags.h>
+#include "kudu/common/column_predicate.h"
+#include "kudu/common/encoded_key.h"
#include "kudu/common/iterator.h"
#include "kudu/common/scan_spec.h"
#include "kudu/common/schema.h"
@@ -32,9 +35,11 @@
#include "kudu/gutil/stl_util.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/tablet/tablet.h"
+#include "kudu/tablet/tablet_metadata.h"
#include "kudu/tablet/tablet_metrics.h"
#include "kudu/tserver/scanner_metrics.h"
#include "kudu/util/flag_tags.h"
+#include "kudu/util/logging.h"
#include "kudu/util/metrics.h"
#include "kudu/util/status.h"
#include "kudu/util/thread.h"
@@ -47,6 +52,11 @@ DEFINE_int32(scanner_gc_check_interval_us, 5 * 1000L *1000L, // 5 seconds
"Number of microseconds in the interval at which we remove expired scanners");
TAG_FLAG(scanner_gc_check_interval_us, hidden);
+DEFINE_int32(scan_history_count, 20,
+ "Number of completed scans to keep history for. Determines how many historical "
+ "scans will be shown on the tablet server's scans dashboard.");
+TAG_FLAG(scan_history_count, experimental);
+
METRIC_DEFINE_gauge_size(server, active_scanners,
"Active Scanners",
kudu::MetricUnit::kScanners,
@@ -64,7 +74,8 @@ namespace tserver {
ScannerManager::ScannerManager(const scoped_refptr<MetricEntity>& metric_entity)
: shutdown_(false),
- shutdown_cv_(&shutdown_lock_) {
+ shutdown_cv_(&shutdown_lock_),
+ completed_scans_offset_(0) {
if (metric_entity) {
metrics_.reset(new ScannerMetrics(metric_entity));
METRIC_active_scanners.InstantiateFunctionGauge(
@@ -75,6 +86,10 @@ ScannerManager::ScannerManager(const scoped_refptr<MetricEntity>& metric_entity)
for (size_t i = 0; i < kNumScannerMapStripes; i++) {
scanner_maps_.push_back(new ScannerMapStripe());
}
+
+ if (FLAGS_scan_history_count > 0) {
+ completed_scans_.reserve(FLAGS_scan_history_count);
+ }
}
ScannerManager::~ScannerManager() {
@@ -146,9 +161,29 @@ bool ScannerManager::LookupScanner(const string& scanner_id, SharedScanner* scan
}
bool ScannerManager::UnregisterScanner(const string& scanner_id) {
+ ScanDescriptor descriptor;
ScannerMapStripe& stripe = GetStripeByScannerId(scanner_id);
- std::lock_guard<RWMutex> l(stripe.lock_);
- return stripe.scanners_by_id_.erase(scanner_id) > 0;
+ {
+ std::lock_guard<RWMutex> l(stripe.lock_);
+ auto it = stripe.scanners_by_id_.find(scanner_id);
+ if (it == stripe.scanners_by_id_.end()) {
+ return false;
+ }
+
+ bool is_initialized = it->second->IsInitialized();
+ if (is_initialized) {
+ descriptor = it->second->descriptor();
+ descriptor.state = it->second->iter()->HasNext() ? ScanState::kFailed : ScanState::kComplete;
+ }
+ stripe.scanners_by_id_.erase(it);
+ if (!is_initialized) {
+ return true;
+ }
+ }
+
+ std::lock_guard<RWMutex> l(completed_scans_lock_);
+ RecordCompletedScanUnlocked(std::move(descriptor));
+ return true;
}
size_t ScannerManager::CountActiveScanners() const {
@@ -160,19 +195,48 @@ size_t ScannerManager::CountActiveScanners() const {
return total;
}
-void ScannerManager::ListScanners(std::vector<SharedScanner>* scanners) {
+void ScannerManager::ListScanners(std::vector<SharedScanner>* scanners) const {
for (const ScannerMapStripe* stripe : scanner_maps_) {
shared_lock<RWMutex> l(stripe->lock_);
- for (const ScannerMapEntry& se : stripe->scanners_by_id_) {
+ for (const auto& se : stripe->scanners_by_id_) {
scanners->push_back(se.second);
}
}
}
+vector<ScanDescriptor> ScannerManager::ListScans() const {
+ vector<ScanDescriptor> scans;
+ for (const ScannerMapStripe* stripe : scanner_maps_) {
+ shared_lock<RWMutex> l(stripe->lock_);
+ for (const auto& se : stripe->scanners_by_id_) {
+ if (se.second->IsInitialized()) {
+ scans.emplace_back(se.second->descriptor());
+ scans.back().state = ScanState::kActive;
+ }
+ }
+ }
+
+ {
+ shared_lock<RWMutex> l(completed_scans_lock_);
+ scans.insert(scans.end(), completed_scans_.begin(), completed_scans_.end());
+ }
+
+ // TODO(dan): It's possible for a descriptor to be included twice in the
+ // result set if its scanner is concurrently removed from the scanner map.
+
+ // Sort oldest to newest, so that the ordering is consistent across calls.
+ std::sort(scans.begin(), scans.end(), [] (const ScanDescriptor& a, const ScanDescriptor& b) {
+ return a.start_time > b.start_time;
+ });
+
+ return scans;
+}
+
void ScannerManager::RemoveExpiredScanners() {
MonoDelta scanner_ttl = MonoDelta::FromMilliseconds(FLAGS_scanner_ttl_ms);
const MonoTime now = MonoTime::Now();
+ vector<ScanDescriptor> descriptors;
for (ScannerMapStripe* stripe : scanner_maps_) {
std::lock_guard<RWMutex> l(stripe->lock_);
for (auto it = stripe->scanners_by_id_.begin(); it != stripe->scanners_by_id_.end();) {
@@ -191,12 +255,36 @@ void ScannerManager::RemoveExpiredScanners() {
scanner->tablet_id(),
idle_time.ToMilliseconds(),
scanner_ttl.ToMilliseconds());
+ if (scanner->IsInitialized()) {
+ descriptors.emplace_back(scanner->descriptor());
+ }
it = stripe->scanners_by_id_.erase(it);
if (metrics_) {
metrics_->scanners_expired->Increment();
}
}
}
+
+ std::lock_guard<RWMutex> l(completed_scans_lock_);
+ for (auto& descriptor : descriptors) {
+ descriptor.last_access_time = now;
+ descriptor.state = ScanState::kExpired;
+ RecordCompletedScanUnlocked(std::move(descriptor));
+ }
+}
+
+void ScannerManager::RecordCompletedScanUnlocked(ScanDescriptor descriptor) {
+ if (completed_scans_.capacity() == 0) {
+ return;
+ }
+ if (completed_scans_.size() == completed_scans_.capacity()) {
+ completed_scans_[completed_scans_offset_++] = std::move(descriptor);
+ if (completed_scans_offset_ == completed_scans_.capacity()) {
+ completed_scans_offset_ = 0;
+ }
+ } else {
+ completed_scans_.emplace_back(std::move(descriptor));
+ }
}
const std::string Scanner::kNullTabletId = "null tablet";
@@ -254,6 +342,57 @@ void Scanner::GetIteratorStats(vector<IteratorStats>* stats) const {
iter_->GetIteratorStats(stats);
}
+ScanDescriptor Scanner::descriptor() const {
+ // Ignore non-initialized scans. The initializing state is transient, and
+ // handling it correctly is complicated. Since the scanner is initialized we
+ // can assume iter(), spec(), and client_projection_schema() return valid
+ // pointers.
+ CHECK(IsInitialized());
+
+ ScanDescriptor descriptor;
+ descriptor.tablet_id = tablet_id();
+ descriptor.scanner_id = id();
+ descriptor.requestor = requestor_string();
+ descriptor.start_time = start_time_;
+
+ for (const auto& column : client_projection_schema()->columns()) {
+ descriptor.projected_columns.emplace_back(column.name());
+ }
+
+ const auto& tablet_metadata = tablet_replica_->tablet_metadata();
+ descriptor.table_name = tablet_metadata->table_name();
+ if (spec().lower_bound_key()) {
+ descriptor.predicates.emplace_back(
+ Substitute("PRIMARY KEY >= $0", KUDU_REDACT(
+ spec().lower_bound_key()->Stringify(tablet_metadata->schema()))));
+ }
+ if (spec().exclusive_upper_bound_key()) {
+ descriptor.predicates.emplace_back(
+ Substitute("PRIMARY KEY < $0", KUDU_REDACT(
+ spec().exclusive_upper_bound_key()->Stringify(tablet_metadata->schema()))));
+ }
+
+ for (const auto& predicate : spec().predicates()) {
+ descriptor.predicates.emplace_back(predicate.second.ToString());
+ }
+
+ vector<IteratorStats> iterator_stats;
+ GetIteratorStats(&iterator_stats);
+
+ DCHECK_EQ(iterator_stats.size(), iter()->schema().num_columns());
+ for (int col_idx = 0; col_idx < iterator_stats.size(); col_idx++) {
+ descriptor.iterator_stats.emplace_back(iter()->schema().column(col_idx).name(),
+ iterator_stats[col_idx]);
+ }
+
+ {
+ std::lock_guard<simple_spinlock> l(lock_);
+ descriptor.last_call_seq_id = call_seq_id_;
+ descriptor.last_access_time = last_access_time_;
+ }
+
+ return descriptor;
+}
} // namespace tserver
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tserver/scanners.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/scanners.h b/src/kudu/tserver/scanners.h
index ec045bb..775ac4a 100644
--- a/src/kudu/tserver/scanners.h
+++ b/src/kudu/tserver/scanners.h
@@ -55,6 +55,8 @@ class Thread;
namespace tserver {
class Scanner;
+enum class ScanState;
+struct ScanDescriptor;
struct ScannerMetrics;
typedef std::shared_ptr<Scanner> SharedScanner;
@@ -96,7 +98,10 @@ class ScannerManager {
// List all active scanners.
// Note this method will not return a consistent view
// of all active scanners if under concurrent modifications.
- void ListScanners(std::vector<SharedScanner>* scanners);
+ void ListScanners(std::vector<SharedScanner>* scanners) const;
+
+ // List active and recently completed scans.
+ std::vector<ScanDescriptor> ListScans() const;
// Iterate through scanners and remove any which are past their TTL.
void RemoveExpiredScanners();
@@ -110,8 +115,6 @@ class ScannerManager {
typedef std::unordered_map<std::string, SharedScanner> ScannerMap;
- typedef std::pair<std::string, SharedScanner> ScannerMapEntry;
-
struct ScannerMapStripe {
// Lock protecting the scanner map.
mutable RWMutex lock_;
@@ -124,6 +127,9 @@ class ScannerManager {
ScannerMapStripe& GetStripeByScannerId(const std::string& scanner_id);
+ // Adds the scan descriptor to the completed scans FIFO.
+ void RecordCompletedScanUnlocked(ScanDescriptor descriptor);
+
// (Optional) scanner metrics for this instance.
gscoped_ptr<ScannerMetrics> metrics_;
@@ -135,6 +141,11 @@ class ScannerManager {
std::vector<ScannerMapStripe*> scanner_maps_;
+ // completed_scans_ is a FIFO ring buffer of completed scans.
+ mutable RWMutex completed_scans_lock_;
+ std::vector<ScanDescriptor> completed_scans_;
+ size_t completed_scans_offset_;
+
// Generator for scanner IDs.
ObjectIdGenerator oid_generator_;
@@ -282,6 +293,8 @@ class Scanner {
return row_format_flags_;
}
+ ScanDescriptor descriptor() const;
+
private:
friend class ScannerManager;
@@ -339,6 +352,44 @@ class Scanner {
DISALLOW_COPY_AND_ASSIGN(Scanner);
};
+enum class ScanState {
+ // The scan is actively running.
+ kActive,
+ // The scan is complete.
+ kComplete,
+ // The scan failed.
+ kFailed,
+ // The scan timed out due to inactivity.
+ kExpired,
+};
+
+// ScanDescriptor holds information about a scan. The ScanDescriptor can outlive
+// the associated scanner without holding open any of the scanner's resources.
+struct ScanDescriptor {
+ // The tablet ID.
+ std::string tablet_id;
+ // The scanner ID.
+ std::string scanner_id;
+
+ // The scan requestor.
+ std::string requestor;
+
+ // The table name.
+ std::string table_name;
+ // The selected columns.
+ std::vector<std::string> projected_columns;
+ // The scan predicates. Holds both the primary key and column predicates.
+ std::vector<std::string> predicates;
+
+ // The per-column scan stats, paired with the column name.
+ std::vector<std::pair<std::string, IteratorStats>> iterator_stats;
+
+ ScanState state;
+
+ MonoTime start_time;
+ MonoTime last_access_time;
+ uint32_t last_call_seq_id;
+};
} // namespace tserver
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tserver/tablet_service.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_service.cc b/src/kudu/tserver/tablet_service.cc
index 22753d8..cd264f6 100644
--- a/src/kudu/tserver/tablet_service.cc
+++ b/src/kudu/tserver/tablet_service.cc
@@ -22,6 +22,7 @@
#include <cstring>
#include <functional>
#include <memory>
+#include <numeric>
#include <ostream>
#include <string>
#include <type_traits>
@@ -1983,21 +1984,17 @@ Status TabletServiceImpl::HandleContinueScanRequest(const ScanRequestPB* req,
// total that we already reported in a previous scan.
vector<IteratorStats> stats_by_col;
scanner->GetIteratorStats(&stats_by_col);
- IteratorStats total_stats;
- for (const IteratorStats& stats : stats_by_col) {
- total_stats.AddStats(stats);
- }
- IteratorStats delta_stats = total_stats;
- delta_stats.SubtractStats(scanner->already_reported_stats());
+ IteratorStats total_stats = std::accumulate(stats_by_col.begin(),
+ stats_by_col.end(),
+ IteratorStats());
+
+ IteratorStats delta_stats = total_stats - scanner->already_reported_stats();
scanner->set_already_reported_stats(total_stats);
if (tablet) {
- tablet->metrics()->scanner_rows_scanned->IncrementBy(
- rows_scanned);
- tablet->metrics()->scanner_cells_scanned_from_disk->IncrementBy(
- delta_stats.cells_read_from_disk);
- tablet->metrics()->scanner_bytes_scanned_from_disk->IncrementBy(
- delta_stats.bytes_read_from_disk);
+ tablet->metrics()->scanner_rows_scanned->IncrementBy(rows_scanned);
+ tablet->metrics()->scanner_cells_scanned_from_disk->IncrementBy(delta_stats.cells_read);
+ tablet->metrics()->scanner_bytes_scanned_from_disk->IncrementBy(delta_stats.bytes_read);
}
scanner->UpdateAccessTime();
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tserver/tserver_path_handlers.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver_path_handlers.cc b/src/kudu/tserver/tserver_path_handlers.cc
index 3dd602f..7cfc935 100644
--- a/src/kudu/tserver/tserver_path_handlers.cc
+++ b/src/kudu/tserver/tserver_path_handlers.cc
@@ -18,8 +18,6 @@
#include "kudu/tserver/tserver_path_handlers.h"
#include <algorithm>
-#include <cstddef>
-#include <cstdint>
#include <iosfwd>
#include <map>
#include <memory>
@@ -33,14 +31,9 @@
#include <boost/bind.hpp> // IWYU pragma: keep
#include <glog/logging.h>
-#include "kudu/common/column_predicate.h"
#include "kudu/common/common.pb.h"
-#include "kudu/common/encoded_key.h"
-#include "kudu/common/iterator.h"
#include "kudu/common/iterator_stats.h"
#include "kudu/common/partition.h"
-#include "kudu/common/scan_spec.h"
-#include "kudu/common/schema.h"
#include "kudu/common/wire_protocol.pb.h"
#include "kudu/consensus/consensus.pb.h"
#include "kudu/consensus/log_anchor_registry.h"
@@ -74,11 +67,11 @@
#include "kudu/util/url-coding.h"
#include "kudu/util/web_callback_registry.h"
-using kudu::consensus::GetConsensusRole;
+using kudu::MaintenanceManagerStatusPB;
using kudu::consensus::ConsensusStatePB;
+using kudu::consensus::GetConsensusRole;
using kudu::consensus::RaftPeerPB;
using kudu::consensus::TransactionStatusPB;
-using kudu::MaintenanceManagerStatusPB;
using kudu::pb_util::SecureDebugString;
using kudu::pb_util::SecureShortDebugString;
using kudu::tablet::Tablet;
@@ -95,13 +88,16 @@ using std::vector;
using strings::Substitute;
namespace kudu {
+
+class Schema;
+
namespace tserver {
TabletServerPathHandlers::~TabletServerPathHandlers() {
}
Status TabletServerPathHandlers::Register(Webserver* server) {
- server->RegisterPrerenderedPathHandler(
+ server->RegisterPathHandler(
"/scans", "Scans",
boost::bind(&TabletServerPathHandlers::HandleScansPage, this, _1, _2),
true /* styled */, false /* is_on_nav_bar */);
@@ -506,105 +502,103 @@ void TabletServerPathHandlers::HandleConsensusStatusPage(const Webserver::WebReq
consensus->DumpStatusHtml(*output);
}
-void TabletServerPathHandlers::HandleScansPage(const Webserver::WebRequest& /*req*/,
- Webserver::PrerenderedWebResponse* resp) {
- std::ostringstream* output = resp->output;
- *output << "<h1>Scans</h1>\n";
- *output << "<table class='table table-striped'>\n";
- *output << "<thead><tr><th>Tablet id</th><th>Scanner id</th><th>Total time in-flight</th>"
- "<th>Time since last update</th><th>Requestor</th><th>Iterator Stats</th>"
- "<th>Pushed down key predicates</th><th>Other predicates</th></tr></thead>\n";
- *output << "<tbody>\n";
-
- vector<SharedScanner> scanners;
- tserver_->scanner_manager()->ListScanners(&scanners);
- for (const SharedScanner& scanner : scanners) {
- *output << ScannerToHtml(*scanner);
+namespace {
+// Pretty-prints a scan's state.
+const char* ScanStateToString(const ScanState& scan_state) {
+ switch (scan_state) {
+ case ScanState::kActive: return "Active";
+ case ScanState::kComplete: return "Complete";
+ case ScanState::kFailed: return "Failed";
+ case ScanState::kExpired: return "Expired";
+ }
+ LOG(FATAL) << "missing ScanState branch";
+}
+
+// Formats the scan descriptor's pseudo-SQL query string as HTML.
+string ScanQueryHtml(const ScanDescriptor& scan) {
+ string query = "<b>SELECT</b> ";
+ if (scan.projected_columns.empty()) {
+ query.append("COUNT(*)");
+ } else {
+ query.append(JoinMapped(scan.projected_columns, EscapeForHtmlToString, ",<br> "));
+ }
+ query.append("<br> <b>FROM</b> ");
+ if (scan.table_name.empty()) {
+ query.append("<unknown>");
+ } else {
+ query.append(EscapeForHtmlToString(scan.table_name));
+ }
+
+ if (!scan.predicates.empty()) {
+ query.append("<br> <b>WHERE</b> ");
+ query.append(JoinMapped(scan.predicates, EscapeForHtmlToString, "<br> <b>AND</b> "));
}
- *output << "</tbody></table>";
+
+ return query;
}
-string TabletServerPathHandlers::ScannerToHtml(const Scanner& scanner) const {
- std::ostringstream html;
- uint64_t time_in_flight_us =
- (MonoTime::Now() - scanner.start_time()).ToMicroseconds();
- uint64_t time_since_last_access_us =
- scanner.TimeSinceLastAccess(MonoTime::Now()).ToMicroseconds();
+void IteratorStatsToJson(const ScanDescriptor& scan, EasyJson* json) {
- html << Substitute("<tr><td>$0</td><td>$1</td><td>$2 us.</td><td>$3 us.</td><td>$4</td>",
- EscapeForHtmlToString(scanner.tablet_id()), // $0
- EscapeForHtmlToString(scanner.id()), // $1
- time_in_flight_us, time_since_last_access_us, // $2, $3
- EscapeForHtmlToString(scanner.requestor_string())); // $4
+ auto fill_stats = [] (EasyJson& row, const string& column, const IteratorStats& stats) {
+ row["column"] = column;
+ row["bytes_read"] = HumanReadableNumBytes::ToString(stats.bytes_read);
+ row["cells_read"] = HumanReadableInt::ToString(stats.cells_read);
+ row["cblocks_read"] = HumanReadableInt::ToString(stats.cblocks_read);
- if (!scanner.IsInitialized()) {
- html << "<td colspan=\"3\"><not yet initialized></td></tr>";
- return html.str();
+ row["bytes_read_title"] = stats.bytes_read;
+ row["cells_read_title"] = stats.cells_read;
+ row["cblocks_read_title"] = stats.cblocks_read;
+ };
+
+ IteratorStats total_stats;
+ for (const auto& column : scan.iterator_stats) {
+ EasyJson row = json->PushBack(EasyJson::kObject);
+ fill_stats(row, column.first, column.second);
+ total_stats += column.second;
}
- const Schema* projection = &scanner.iter()->schema();
+ EasyJson total_row = json->PushBack(EasyJson::kObject);
+ fill_stats(total_row, "total", total_stats);
+}
- vector<IteratorStats> stats;
- scanner.GetIteratorStats(&stats);
- CHECK_EQ(stats.size(), projection->num_columns());
- html << Substitute("<td>$0</td>", IteratorStatsToHtml(*projection, stats));
- scoped_refptr<TabletReplica> tablet_replica;
- if (!tserver_->tablet_manager()->LookupTablet(scanner.tablet_id(), &tablet_replica)) {
- html << Substitute("<td colspan=\"2\"><b>Tablet $0 is no longer valid.</b></td></tr>\n",
- scanner.tablet_id());
+void ScanToJson(const ScanDescriptor& scan, EasyJson* json) {
+ MonoTime now = MonoTime::Now();
+ MonoDelta duration;
+ if (scan.state == ScanState::kActive) {
+ duration = now - scan.start_time;
} else {
- string range_pred_str;
- vector<string> other_preds;
- const ScanSpec& spec = scanner.spec();
- if (spec.lower_bound_key() || spec.exclusive_upper_bound_key()) {
- range_pred_str = EncodedKey::RangeToString(spec.lower_bound_key(),
- spec.exclusive_upper_bound_key());
- }
- for (const auto& col_pred : scanner.spec().predicates()) {
- int32_t col_idx = projection->find_column(col_pred.first);
- if (col_idx == Schema::kColumnNotFound) {
- other_preds.emplace_back("unknown column");
- } else {
- other_preds.push_back(col_pred.second.ToString());
- }
- }
- string other_pred_str = JoinStrings(other_preds, "\n");
- html << Substitute("<td>$0</td><td>$1</td></tr>\n",
- EscapeForHtmlToString(range_pred_str),
- EscapeForHtmlToString(other_pred_str));
+ duration = scan.last_access_time - scan.start_time;
}
- return html.str();
+ MonoDelta time_since_start = now - scan.start_time;
+
+ json->Set("tablet_id", scan.tablet_id);
+ json->Set("scanner_id", scan.scanner_id);
+ json->Set("state", ScanStateToString(scan.state));
+ json->Set("query", ScanQueryHtml(scan));
+ json->Set("requestor", scan.requestor);
+
+ json->Set("duration", HumanReadableElapsedTime::ToShortString(duration.ToSeconds()));
+ json->Set("time_since_start",
+ HumanReadableElapsedTime::ToShortString(time_since_start.ToSeconds()));
+
+ json->Set("duration_title", duration.ToSeconds());
+ json->Set("time_since_start_title", time_since_start.ToSeconds());
+
+ EasyJson stats_json = json->Set("stats", EasyJson::kArray);
+ IteratorStatsToJson(scan, &stats_json);
}
+} // anonymous namespace
+
+void TabletServerPathHandlers::HandleScansPage(const Webserver::WebRequest& /*req*/,
+ Webserver::WebResponse* resp) {
+ EasyJson scans = resp->output->Set("scans", EasyJson::kArray);
+ vector<ScanDescriptor> descriptors = tserver_->scanner_manager()->ListScans();
-string TabletServerPathHandlers::IteratorStatsToHtml(const Schema& projection,
- const vector<IteratorStats>& stats) const {
- std::ostringstream html;
- html << "<table>\n";
- html << "<tr><th>Column</th>"
- << "<th>Blocks read from disk</th>"
- << "<th>Bytes read from disk</th>"
- << "<th>Cells read from disk</th>"
- << "</tr>\n";
- for (size_t idx = 0; idx < stats.size(); idx++) {
- // We use 'title' attributes so that if the user hovers over the value, they get a
- // human-readable tooltip.
- html << Substitute("<tr>"
- "<td>$0</td>"
- "<td title=\"$1\">$2</td>"
- "<td title=\"$3\">$4</td>"
- "<td title=\"$5\">$6</td>"
- "</tr>\n",
- EscapeForHtmlToString(projection.column(idx).name()), // $0
- HumanReadableInt::ToString(stats[idx].data_blocks_read_from_disk), // $1
- stats[idx].data_blocks_read_from_disk, // $2
- HumanReadableNumBytes::ToString(stats[idx].bytes_read_from_disk), // $3
- stats[idx].bytes_read_from_disk, // $4
- HumanReadableInt::ToString(stats[idx].cells_read_from_disk), // $5
- stats[idx].cells_read_from_disk); // $6
+ for (const auto& descriptor : descriptors) {
+ EasyJson scan = scans.PushBack(EasyJson::kObject);
+ ScanToJson(descriptor, &scan);
}
- html << "</table>\n";
- return html.str();
}
void TabletServerPathHandlers::HandleDashboardsPage(const Webserver::WebRequest& /*req*/,
@@ -614,7 +608,8 @@ void TabletServerPathHandlers::HandleDashboardsPage(const Webserver::WebRequest&
*output << "<table class='table table-striped'>\n";
*output << " <thead><tr><th>Dashboard</th><th>Description</th></tr></thead>\n";
*output << " <tbody\n";
- *output << GetDashboardLine("scans", "Scans", "List of scanners that are currently running.");
+ *output << GetDashboardLine("scans", "Scans", "List of currently running and recently "
+ "completed scans.");
*output << GetDashboardLine("transactions", "Transactions", "List of transactions that are "
"currently running.");
*output << GetDashboardLine("maintenance-manager", "Maintenance Manager",
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/src/kudu/tserver/tserver_path_handlers.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver_path_handlers.h b/src/kudu/tserver/tserver_path_handlers.h
index 8c00a6f..cb7c88f 100644
--- a/src/kudu/tserver/tserver_path_handlers.h
+++ b/src/kudu/tserver/tserver_path_handlers.h
@@ -18,7 +18,6 @@
#define KUDU_TSERVER_TSERVER_PATH_HANDLERS_H
#include <string>
-#include <vector>
#include "kudu/gutil/macros.h"
#include "kudu/server/webserver.h"
@@ -26,9 +25,6 @@
namespace kudu {
-class Schema;
-struct IteratorStats;
-
namespace consensus {
class ConsensusStatePB;
} // namespace consensus
@@ -36,7 +32,6 @@ class ConsensusStatePB;
namespace tserver {
class TabletServer;
-class Scanner;
class TabletServerPathHandlers {
public:
@@ -50,7 +45,7 @@ class TabletServerPathHandlers {
private:
void HandleScansPage(const Webserver::WebRequest& req,
- Webserver::PrerenderedWebResponse* resp);
+ Webserver::WebResponse* resp);
void HandleTabletsPage(const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp);
void HandleTabletPage(const Webserver::WebRequest& req,
@@ -68,9 +63,6 @@ class TabletServerPathHandlers {
void HandleMaintenanceManagerPage(const Webserver::WebRequest& req,
Webserver::WebResponse* resp);
std::string ConsensusStatePBToHtml(const consensus::ConsensusStatePB& cstate) const;
- std::string ScannerToHtml(const Scanner& scanner) const;
- std::string IteratorStatsToHtml(const Schema& projection,
- const std::vector<IteratorStats>& stats) const;
std::string GetDashboardLine(const std::string& link,
const std::string& text, const std::string& desc);
http://git-wip-us.apache.org/repos/asf/kudu/blob/5356334e/www/scans.mustache
----------------------------------------------------------------------
diff --git a/www/scans.mustache b/www/scans.mustache
new file mode 100644
index 0000000..3c6edad
--- /dev/null
+++ b/www/scans.mustache
@@ -0,0 +1,72 @@
+{{!
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+}}{{#raw}}{{{.}}}{{/raw}}{{^raw}}
+
+<h1>Scans</h1>
+<table class="table table-striped">
+ <thead>
+ <tr>
+ <th>Tablet id</th>
+ <th>Scanner id</th>
+ <th>State</th>
+ <th title="pseudo-SQL query description">Query</th>
+ <th>Requestor</th>
+ <th title="running time of the scan">Duration</th>
+ <th title="elapsed time since the scan started">Time since start</th>
+ <th>Column Stats</th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#scans}}
+ <tr>
+ <td><a href="/tablet?id={{tablet_id}}"><samp>{{tablet_id}}</samp></a></td>
+ <td><samp>{{scanner_id}}</samp></td>
+ <td>{{state}}</td>
+ {{! The query string is pre-formatted HTML, so don't escape it (triple-brace). }}
+ <td><pre>{{{query}}}</pre></td>
+ <td><samp>{{requestor}}</samp></td>
+ <td title="{{duration_title}}">{{duration}}</td>
+ <td title="{{time_since_start_title}}">{{time_since_start}}</td>
+
+ <td>
+ <table class="table table-striped">
+ <thead>
+ <tr>
+ <th>column</th>
+ <th title="cells read from the column (disk or cache), exclusive of the MRS">cells read</th>
+ <th title="bytes read from the column (disk or cache), exclusive of the MRS">bytes read</th>
+ <th title="CFile data blocks read from the column (disk or cache)">cblocks read</th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#stats}}
+ <tr>
+ <td>{{column}}</td>
+ <td title="{{cells_read_title}}">{{cells_read}}</td>
+ <td title="{{bytes_read_title}}">{{bytes_read}}</td>
+ <td title="{{cblocks_read_title}}">{{cblocks_read}}</td>
+ </tr>
+ {{/stats}}
+ </tbody>
+ </table>
+ </td>
+ </tr>
+ {{/scans}}
+ </tbody>
+</table>
+{{/raw}}