You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by wd...@apache.org on 2018/10/09 17:07:00 UTC
[2/4] kudu git commit: Cleanup in rowset SVG layout dumping
Cleanup in rowset SVG layout dumping
I tested this by manually verifying that the /tablet-rowsetlayout-svg
endpoint and rowset layout SVG file dumping worked as expected.
Change-Id: I521fadf4336f9e11ea611b022cd4d8fcb269dfe2
Reviewed-on: http://gerrit.cloudera.org:8080/11612
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Attila Bukor <ab...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/77ead354
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/77ead354
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/77ead354
Branch: refs/heads/master
Commit: 77ead354b6ac86e3d251e35d9b0cf9da62979054
Parents: 891a8e3
Author: Will Berkeley <wd...@gmail.org>
Authored: Sun Oct 7 21:56:14 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Tue Oct 9 15:10:20 2018 +0000
----------------------------------------------------------------------
src/kudu/tablet/compaction_policy.cc | 2 +-
src/kudu/tablet/svg_dump.cc | 172 ++++++++++++++----------------
src/kudu/tablet/svg_dump.h | 23 ++--
src/kudu/tablet/tablet.cc | 2 +-
4 files changed, 95 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/77ead354/src/kudu/tablet/compaction_policy.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/compaction_policy.cc b/src/kudu/tablet/compaction_policy.cc
index 745f28b..d1ead15 100644
--- a/src/kudu/tablet/compaction_policy.cc
+++ b/src/kudu/tablet/compaction_policy.cc
@@ -471,7 +471,7 @@ Status BudgetedCompactionPolicy::PickRowSets(const RowSetTree &tree,
}
picked->swap(best_solution.rowsets);
- DumpCompactionSVG(asc_min_key, *picked);
+ DumpCompactionSVGToFile(asc_min_key, *picked);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/77ead354/src/kudu/tablet/svg_dump.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/svg_dump.cc b/src/kudu/tablet/svg_dump.cc
index 1427b9b..9c88655 100644
--- a/src/kudu/tablet/svg_dump.cc
+++ b/src/kudu/tablet/svg_dump.cc
@@ -17,8 +17,8 @@
#include "kudu/tablet/svg_dump.h"
+#include <algorithm>
#include <ctime>
-#include <fstream>
#include <string>
#include <unordered_set>
#include <vector>
@@ -26,13 +26,16 @@
#include <gflags/gflags.h>
#include <glog/logging.h>
-#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/macros.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/stringprintf.h"
+#include "kudu/gutil/strings/substitute.h"
#include "kudu/gutil/strings/util.h"
#include "kudu/tablet/rowset_info.h"
+#include "kudu/util/env.h"
#include "kudu/util/flag_tags.h"
+#include "kudu/util/slice.h"
+#include "kudu/util/status.h"
// Flag to dump SVGs of every compaction decision.
//
@@ -40,7 +43,6 @@
// commands like:
// $ for x in compaction-*svg ; do convert $x $x.png ; done
// $ mencoder mf://compaction*png -mf fps=1 -ovc lavc -o compactions.avi
-
DEFINE_string(compaction_policy_dump_svgs_pattern, "",
"File path into which to dump SVG visualization of "
"selected compactions. This is mostly useful in "
@@ -49,10 +51,12 @@ DEFINE_string(compaction_policy_dump_svgs_pattern, "",
"with the compaction selection timestamp.");
TAG_FLAG(compaction_policy_dump_svgs_pattern, hidden);
+using std::endl;
using std::ostream;
using std::string;
using std::unordered_set;
using std::vector;
+using strings::Substitute;
namespace kudu {
namespace tablet {
@@ -63,35 +67,30 @@ namespace {
// distributes 'rowsets' into separate vectors in 'rows' such that
// within any given row, none of the rowsets overlap in keyspace.
void OrganizeSVGRows(const vector<RowSetInfo>& candidates,
- vector<vector<const RowSetInfo*> >* rows) {
- rows->push_back(vector<const RowSetInfo *>());
-
- for (const RowSetInfo &candidate : candidates) {
- // Slot into the first row of the output which fits it
+ vector<vector<const RowSetInfo*>>* rows) {
+ DCHECK(rows);
+ rows->clear();
+ for (const auto& candidate : candidates) {
+ // Slot into the first row which fits it.
bool found_slot = false;
- for (vector<const RowSetInfo *> &row : *rows) {
- // If this candidate doesn't intersect any other candidates in this
- // row, we can put it here.
- bool fits_in_row = true;
- for (const RowSetInfo *already_in_row : row) {
- if (candidate.Intersects(*already_in_row)) {
- fits_in_row = false;
- break;
- }
- }
+ for (auto& row : *rows) {
+ // If this candidate doesn't intersect any other rowsets already in this
+ // row, we can put it in this row.
+ auto fits_in_row = std::none_of(row.begin(),
+ row.end(),
+ [&candidate](const RowSetInfo* already_in_row) {
+ return candidate.Intersects(*already_in_row);
+ });
if (fits_in_row) {
row.push_back(&candidate);
found_slot = true;
break;
}
}
-
// If we couldn't find a spot in any existing row, add a new row
// to the bottom of the SVG.
if (!found_slot) {
- vector<const RowSetInfo *> new_row;
- new_row.push_back(&candidate);
- rows->push_back(new_row);
+ rows->push_back({ &candidate });
}
}
}
@@ -99,79 +98,62 @@ void OrganizeSVGRows(const vector<RowSetInfo>& candidates,
void DumpSVG(const vector<RowSetInfo>& candidates,
const unordered_set<RowSet*>& picked,
ostream* outptr) {
- CHECK(outptr) << "Dump SVG expects an ostream";
- CHECK(outptr->good()) << "Dump SVG expects a good ostream";
- using std::endl;
+ CHECK(outptr);
+ CHECK(outptr->good());
ostream& out = *outptr;
- vector<vector<const RowSetInfo*> > svg_rows;
+ vector<vector<const RowSetInfo*>> svg_rows;
OrganizeSVGRows(candidates, &svg_rows);
- const char *kPickedColor = "#f66";
- const char *kDefaultColor = "#666";
- const double kTotalWidth = 1200;
- const int kRowHeight = 15;
- const double kHeaderHeight = 60;
+ const char *kPickedColor = "#f66"; // Light red.
+ const char *kDefaultColor = "#666"; // Dark gray.
+ constexpr double kTotalWidth = 1200.0;
+ constexpr int kRowHeight = 15;
+ constexpr double kHeaderHeight = 60.0;
const double kTotalHeight = kRowHeight * svg_rows.size() + kHeaderHeight;
- out << "<svg version=\"1.1\" width=\"" << kTotalWidth << "\" height=\""
- << kTotalHeight << "\""
- << " viewBox=\"0 0 " << kTotalWidth << " " << kTotalHeight << "\""
- << " xmlns=\"http://www.w3.org/2000/svg\" >" << endl;
+ out << Substitute(
+ R"(<svg version="1.1" width="$0" height="$1" viewBox="0 0 $0 $1" )"
+ R"(xmlns="http://www.w3.org/2000/svg">)",
+ kTotalWidth, kTotalHeight)
+ << endl;
- // Background
- out << "<rect x=\"0.0\" y=\"0\" width=\"1200.0\" height=\"" << kTotalHeight << "\""
- << " fill=\"#fff\" />" << endl;
+ // Background.
+ out << Substitute(R"(<rect x="0" y="0" width="$0" height="$1" fill="#fff"/>)",
+ kTotalWidth, kTotalHeight)
+ << endl;
- for (int row_index = 0; row_index < svg_rows.size(); row_index++) {
+ for (auto row_index = 0; row_index < svg_rows.size(); row_index++) {
const vector<const RowSetInfo *> &row = svg_rows[row_index];
- int y = kRowHeight * row_index + kHeaderHeight;
+ const auto y = kRowHeight * row_index + kHeaderHeight;
for (const RowSetInfo *cand : row) {
- bool was_picked = ContainsKey(picked, cand->rowset());
- const char *color = was_picked ? kPickedColor : kDefaultColor;
-
- double x = cand->cdf_min_key() * kTotalWidth;
- double width = cand->width() * kTotalWidth;
- out << StringPrintf("<rect x=\"%f\" y=\"%d\" width=\"%f\" height=\"%d\" "
- "stroke=\"#000\" fill=\"%s\"/>",
- x, y, width, kRowHeight, color) << endl;
- out << StringPrintf("<text x=\"%f\" y=\"%d\" width=\"%f\" height=\"%d\" "
- "fill=\"rgb(0,0,0)\">%dMB</text>",
- x, y + kRowHeight, width, kRowHeight, cand->size_mb()) << endl;
+ const char *color = ContainsKey(picked, cand->rowset()) ? kPickedColor :
+ kDefaultColor;
+ const auto x = cand->cdf_min_key() * kTotalWidth;
+ const auto width = cand->width() * kTotalWidth;
+ out << Substitute(
+ R"(<rect x="$0" y="$1" width="$2" height="$3" stroke="#000" fill="$4"/>)",
+ x, y, width, kRowHeight, color)
+ << endl;
+ out << Substitute(R"+(<text x="$0" y="$1" width="$2" height="$3" )+"
+ R"+(fill="rgb(0,0,0)">$4MB</text>)+",
+ x, y + kRowHeight, width, kRowHeight, cand->size_mb())
+ << endl;
}
}
out << "</svg>" << endl;
}
-void PrintXMLHeader(ostream* o) {
- CHECK(o) << "XML header printer expects an ostream";
- CHECK(o->good()) << "XML header printer expects a good ostream";
- *o << "<?xml version=\"1.0\" standalone=\"no\"?>" << std::endl;
- *o << "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\" "
- << "\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">" << std::endl;
-}
-
-// Prepares ofstream to default dump location.
-// In case any of the preparation fails or default pattern is empty,
-// NULL is returned.
-gscoped_ptr<ostream> PrepareOstream() {
- using std::ofstream;
- gscoped_ptr<ofstream> out;
- // Get default file name
- const string &pattern = FLAGS_compaction_policy_dump_svgs_pattern;
- if (pattern.empty()) return gscoped_ptr<ostream>();
- const string path = StringReplace(pattern, "TIME", StringPrintf("%ld", time(nullptr)), true);
-
- // Open
- out.reset(new ofstream(path.c_str()));
- if (!out->is_open()) {
- LOG(WARNING) << "Could not dump compaction output to " << path << ": file open failed";
- return gscoped_ptr<ostream>();
- }
-
- return out.PassAs<ostream>();
+void PrintXMLHeader(ostream* out) {
+ CHECK(out);
+ CHECK(out->good());
+ *out << R"(<?xml version="1.0" standalone="no"?>)"
+ << endl
+ << R"(<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" )"
+ << R"("http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">)"
+ << endl;
}
} // anonymous namespace
@@ -179,23 +161,31 @@ gscoped_ptr<ostream> PrepareOstream() {
void DumpCompactionSVG(const vector<RowSetInfo>& candidates,
const unordered_set<RowSet*>& picked,
ostream* out,
- bool print_xml) {
- // Get the desired pointer to the ostream
- gscoped_ptr<ostream> dfl;
- if (!out) {
- dfl = PrepareOstream();
- out = dfl.get();
- if (!out) return;
- }
-
- // Print out with the correct ostream
- LOG(INFO) << "Dumping SVG of DiskRowSetLayout with"
- << (print_xml ? "" : "out") << " XML header";
- if (print_xml) {
+ bool print_xml_header) {
+ CHECK(out);
+ VLOG(1) << "Dumping SVG of DiskRowSetLayout with"
+ << (print_xml_header ? "" : "out") << " XML header";
+ if (print_xml_header) {
PrintXMLHeader(out);
}
DumpSVG(candidates, picked, out);
}
+void DumpCompactionSVGToFile(const vector<RowSetInfo>& candidates,
+ const unordered_set<RowSet*>& picked) {
+ const string& pattern = FLAGS_compaction_policy_dump_svgs_pattern;
+ if (pattern.empty()) {
+ return;
+ }
+ const string path = StringReplace(pattern,
+ "TIME",
+ StringPrintf("%ld", time(nullptr)),
+ /*replace_all=*/true);
+ std::ostringstream buf;
+ DumpCompactionSVG(candidates, picked, &buf, /*print_xml_header=*/true);
+ WARN_NOT_OK(WriteStringToFile(Env::Default(), buf.str(), path),
+ "unable to dump rowset compaction SVG to file");
+}
+
} // namespace tablet
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/77ead354/src/kudu/tablet/svg_dump.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/svg_dump.h b/src/kudu/tablet/svg_dump.h
index 59c1397..706f573 100644
--- a/src/kudu/tablet/svg_dump.h
+++ b/src/kudu/tablet/svg_dump.h
@@ -17,7 +17,6 @@
#ifndef KUDU_TABLET_COMPACTION_SVG_DUMP_H_
#define KUDU_TABLET_COMPACTION_SVG_DUMP_H_
-#include <cstddef>
#include <ostream>
#include <unordered_set>
#include <vector>
@@ -26,20 +25,22 @@ namespace kudu {
namespace tablet {
class RowSet;
-
class RowSetInfo;
-// Dump an SVG file which represents the candidates
-// for compaction, highlighting the ones that were selected.
-// Dumps in to parameter ostream. If ostream is null, then default ostream
-// specified as a flag is used (see svg_dump.cc).
-// The last optional parameter controls whether to print an XML header in
-// the file. If true, prints the header (xml tag and DOCTYPE). Otherwise, only
-// the <svg>...</svg> section is printed.
+// Dumps an SVG file which describes the rowset layout for the tablet replica
+// and which highlights rowsets that would be selected for the next rowset
+// compaction, if one were run. The SVG is printed to 'out', which must not be
+// null. If 'print_xml_header' is true, prints an XML header including the xml
+// tag and DOCTYPE. Otherwise, only the '<svg>...</svg>' section is printed.
void DumpCompactionSVG(const std::vector<RowSetInfo>& candidates,
const std::unordered_set<RowSet*>& picked,
- std::ostream* out = NULL,
- bool print_xml = true);
+ std::ostream* out,
+ bool print_xml_header);
+
+// Like the above, but dumps the SVG to a file named according to the rules of
+// --compaction_policy_dump_svgs_pattern. See the flag definition in svg_dump.cc.
+void DumpCompactionSVGToFile(const std::vector<RowSetInfo>& candidates,
+ const std::unordered_set<RowSet*>& picked);
} // namespace tablet
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/77ead354/src/kudu/tablet/tablet.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 4fa272f..ac18d41 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -2306,7 +2306,7 @@ void Tablet::PrintRSLayout(ostream* o) {
vector<RowSetInfo> min, max;
RowSetInfo::CollectOrdered(*rowsets_copy, &min, &max);
- DumpCompactionSVG(min, picked, o, false);
+ DumpCompactionSVG(min, picked, o, /*print_xml_header=*/false);
*o << "<h2>Compaction policy log</h2>" << std::endl;