You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bi...@apache.org on 2018/10/09 23:18:22 UTC
[6/8] impala git commit: Prettify the timeline produced by
test-with-docker.py
Prettify the timeline produced by test-with-docker.py
The change tweaks the HTML template for the timeline summary
to make it slightly more readable:
- Adds legend strings to the CPU graphs
- Inserts the test run name into the CPU chart title to clarify
which chart show which build/test phase
- Stretches the CPU charts a bit wider
- Identifes the common prefix of the phase/container names (the build
name) and delete it from the chart labels. This increases legibility
by cutting down on noise and growing the chart real estate.
To support this change the Python drivers are also changed:
the build name parameter, which is the common prefix, is passed
to monitor.py and written to the JSON output
- The name of the build and data load phase container is suffixed with
"-build" so that it shares the naming convention for the other
containers.
- The timeline graph section is sized explicitly byt computing the height
from the number of distinct tasks. This avoids having a second scrollbar
for the timeline, which is annoying.
The formula is pretty crude: it uses empirical constants, but produces
an OK layout for the default font sizes in Chrome (both on Linux
and the Mac).
Tested so far by tweaking the HTML template and an HTML result file
from an earlier build.
Change-Id: I7a41bea762b0e33f3d71b0be57eedbacb19c680c
Reviewed-on: http://gerrit.cloudera.org:8080/11578
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ee3da437
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ee3da437
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ee3da437
Branch: refs/heads/master
Commit: ee3da43709895450bded45a6aa9bccca947c6add
Parents: 122c366
Author: Laszlo Gaal <la...@cloudera.com>
Authored: Wed Sep 26 15:17:27 2018 +0200
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 9 19:12:50 2018 +0000
----------------------------------------------------------------------
docker/monitor.py | 7 +++---
docker/test-with-docker.py | 5 ++--
docker/timeline.html.template | 47 +++++++++++++++++++++++++++++++-------
3 files changed, 46 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/ee3da437/docker/monitor.py
----------------------------------------------------------------------
diff --git a/docker/monitor.py b/docker/monitor.py
index 58e2a83..5eefe5c 100644
--- a/docker/monitor.py
+++ b/docker/monitor.py
@@ -217,10 +217,11 @@ class Timeline(object):
the charts, which happens in the browser.
"""
- def __init__(self, monitor_file, containers, interesting_re):
+ def __init__(self, monitor_file, containers, interesting_re, buildname):
self.monitor_file = monitor_file
self.containers = containers
self.interesting_re = interesting_re
+ self.buildname = buildname
def logfile_timeline(self, container):
"""Returns a list of (name, timestamp, line) tuples for interesting lines in
@@ -340,7 +341,7 @@ class Timeline(object):
template_path = os.path.join(os.path.dirname(__file__), "timeline.html.template")
shutil.copyfileobj(file(template_path), o)
o.write("\n<script>\nvar data = \n")
- json.dump(dict(timeline=timeline_json, metrics=metrics_by_container,
- max_ts=(max_metric_ts - min_ts)), o, indent=2)
+ json.dump(dict(buildname=self.buildname, timeline=timeline_json,
+ metrics=metrics_by_container, max_ts=(max_metric_ts - min_ts)), o, indent=2)
o.write("</script>")
o.close()
http://git-wip-us.apache.org/repos/asf/impala/blob/ee3da437/docker/test-with-docker.py
----------------------------------------------------------------------
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index dbeef81..4efd712 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -616,7 +616,7 @@ class TestWithDocker(object):
def _create_build_image(self):
"""Creates the "build image", with Impala compiled and data loaded."""
container = self._create_container(
- image="ubuntu:16.04", name=self.name,
+ image="ubuntu:16.04", name=self.name + "-build",
logdir="build",
logname="log-build.txt",
# entrypoint.sh will create a user with our uid; this
@@ -709,7 +709,8 @@ class TestWithDocker(object):
timeline = monitor.Timeline(
monitor_file=self.monitoring_output_file,
containers=self.containers,
- interesting_re=self._INTERESTING_RE)
+ interesting_re=self._INTERESTING_RE,
+ buildname=self.name)
timeline.create(os.path.join(self.log_dir, "timeline.html"))
def log_summary(self):
http://git-wip-us.apache.org/repos/asf/impala/blob/ee3da437/docker/timeline.html.template
----------------------------------------------------------------------
diff --git a/docker/timeline.html.template b/docker/timeline.html.template
index d2960d7..2add9a6 100644
--- a/docker/timeline.html.template
+++ b/docker/timeline.html.template
@@ -104,32 +104,61 @@ function drawChart() {
// timeofday isn't supported here
dataTable.addColumn({ type: 'datetime', id: 'Start' });
dataTable.addColumn({ type: 'datetime', id: 'End' });
+
+ // Extract the build name, which is a common prefix of the container names.
+ // Add one to the length we know that the driver script adds a hyphen when building
+ //the container names.
+ var buildnamePrefix = data.buildname;
+ var commonPrefix = buildnamePrefix.length + 1;
// Timeline
+ const rowLabels = new Set();
for (i = 0; i < data.timeline.length; ++i) {
var row = data.timeline[i];
- dataTable.addRow([ row[0], row[1], ts_to_date(row[2]), ts_to_date(row[3]) ]);
+ // Hack up the label in row[0]: cut off the common prefix, which is the build
+ // name. This is the same for all shards, so it is not useful for the charts.
+ var rowLabel = row[0].slice(commonPrefix);
+ if (rowLabel == "") rowLabel = ' -- ';
+ dataTable.addRow([ rowLabel, row[1], ts_to_date(row[2]), ts_to_date(row[3]) ]);
+ rowLabels.add(rowLabel);
}
- chart.draw(dataTable, { height: "400px" } );
+ // Precompute timeline height to get rid of the timeline's own scrollbar.
+ const laneHeight = 41;
+ const axisHeight = 50;
+
+ timelineHeight = rowLabels.size * laneHeight + axisHeight;
+ var options = {
+ // Width is specified in the <div> tag below; for some reason it doesn't have the
+ // same effect when specified here.
+ // Height is precomputed from the incoming data, so we set it here.
+ height: timelineHeight,
+ timeline: { showRowLabels : 'true' }
+ };
+ chart.draw(dataTable, options);
for (const k of Object.keys(data.metrics)) {
var lineChart = document.createElement("div");
lineChartContainer.appendChild(lineChart);
var dataTable = new google.visualization.DataTable();
- dataTable.addColumn({ type: 'timeofday', id: 'Time' });
- dataTable.addColumn({ type: 'number', id: 'User' });
- dataTable.addColumn({ type: 'number', id: 'System' });
+ dataTable.addColumn({ type: 'timeofday', label: 'Time', id: 'Time' });
+ dataTable.addColumn({ type: 'number', label: 'User', id: 'User' });
+ dataTable.addColumn({ type: 'number', label: 'System', id: 'System' });
for (const row of data.metrics[k]) {
dataTable.addRow([ ts_to_hms(row[0]), row[1], row[2] ]);
}
+ // De-clutter the timeline labels: Slice off the common prefix of the build name
+ var title = k.slice(commonPrefix);
+ if (title == "") title = ' -- ';
var options = {
- title: 'CPU',
+ title: 'CPU: ' + title,
legend: { position: 'bottom' },
hAxis: {
minValue: [0, 0, 0],
maxValue: ts_to_hms(data.max_ts)
- }
+ },
+ // Stretch the CPU graph a bit wider than the default
+ chartArea: {width: '95%'},
};
var chart = new google.visualization.LineChart(lineChart);
@@ -137,5 +166,7 @@ function drawChart() {
}
}
</script>
-<div id="timelineContainer" style="height: 400px;"></div>
+<!-- Set width here, but leave height unset,
+ it is computed above in the JavaScript code. -->
+<div id="timelineContainer" style="width: 99%;"></div>
<div id="lineChartContainer" style="height: 200px;"></div>