You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2016/02/26 11:00:58 UTC
hive git commit: HIVE-13122: LLAP: simple Model/View separation for
UI (Gopal V)
Repository: hive
Updated Branches:
refs/heads/master 4984714d5 -> 5c078940e
HIVE-13122: LLAP: simple Model/View separation for UI (Gopal V)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5c078940
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5c078940
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5c078940
Branch: refs/heads/master
Commit: 5c078940ee9d115abd3eaabb7273cc921c8e394a
Parents: 4984714
Author: Gopal V <go...@apache.org>
Authored: Fri Feb 26 02:00:44 2016 -0800
Committer: Gopal V <go...@apache.org>
Committed: Fri Feb 26 02:00:44 2016 -0800
----------------------------------------------------------------------
.../main/resources/hive-webapps/llap/index.html | 72 ++++-
.../resources/hive-webapps/llap/js/metrics.js | 279 +++++++++++++------
2 files changed, 265 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5c078940/llap-server/src/main/resources/hive-webapps/llap/index.html
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/hive-webapps/llap/index.html b/llap-server/src/main/resources/hive-webapps/llap/index.html
index 3a7575e..31a27a0 100644
--- a/llap-server/src/main/resources/hive-webapps/llap/index.html
+++ b/llap-server/src/main/resources/hive-webapps/llap/index.html
@@ -47,28 +47,35 @@
<div class="page-header">
<h1>LLAP Monitor</h1>
</div>
- <h2 id="hostname"></h2>
+ <h2 id="hostname"></h2>
</div>
<div class="row">
<section id="heap">
<h2>Heap Metrics</h2>
- <table id="attributes_table" class="table table-striped table">
+ <table id="heap_metrics_table" class="table table-striped table">
<thead>
<tr>
<th>Used(MB)</th>
<th>Max(MB)</th>
<th>Use Rate(%)</th>
+ <th>GC time (seconds) </th>
</tr>
</thead>
<tbody id="heap_body">
+ <tr>
+ <td> <span id="heap-used" /> </td>
+ <td> <span id="heap-max" /> </td>
+ <td> <span id="heap-rate"> </span> <span id="heap-trend" /></td>
+ <td> <span id="heap-gc"> </span> <span id="heap-gc-trend" /></td>
+ </tr>
</tbody>
</table>
</section>
<section id="cache">
<h2>Cache Metrics</h2>
- <table id="attributes_table" class="table table-striped">
+ <table id="cache_metrics_table" class="table table-striped">
<thead>
<tr>
<th>Used(MB)</th>
@@ -79,13 +86,20 @@
</tr>
</thead>
<tbody id="cache_body">
+ <tr>
+ <td> <span id="cache-used" /> </td>
+ <td> <span id="cache-max" /> </td>
+ <td> <span id="cache-fill-rate" /> <span id="cache-fill-trend"/></td>
+ <td> <span id="cache-requests" /> </td>
+ <td> <span id="cache-hits"> </span> <span id="cache-hits-trend"/></td>
+ </tr>
</tbody>
</table>
</section>
<section id="executors">
<h2>Executors</h2>
- <table id="attributes_table" class="table table-striped">
+ <table id="executor_metrics_table" class="table table-striped">
<thead>
<tr>
<th>Used</th>
@@ -93,10 +107,60 @@
<th>Use Rate(%)</th>
<th>Queue</th>
<th>Executing+Queuing Tasks</th>
+ </tr>
+ </thead>
+ <tbody id="executors_body">
+ <tr>
+ <td> <span id="executors-used" /> </td>
+ <td> <span id="executors-max" /> </td>
+ <td> <span id="executors-rate"> </span> <span id="executors-trend"/></td>
+ <td> <span id="executors-queue" /> </td>
+ <td> <span id="executors-pending" > </span> <span id="executors-pending-trend"/></td>
+ </tr>
+ </tbody>
+ </table>
+ </section>
+
+ <section id="Fragments">
+ <h2>Fragments</h2>
+ <table id="fragment_metrics_table" class="table table-striped">
+ <thead>
+ <tr>
+ <th>Total Fragments</th>
+ <th>Failed Fragments</th>
+ <th>Preempted Fragments</th>
<th>Preemption Time Lost(s)</th>
</tr>
</thead>
+ <tbody id="fragments_body">
+ <tr>
+ <td> <span id="fragments-total" /> </td>
+ <td> <span id="fragments-failed" /> </td>
+ <td> <span id="fragments-preempted"> </span> </td>
+ <td> <span id="fragments-preemption-time" /> </td>
+ </tr>
+ </tbody>
+ </table>
+ </section>
+
+ <section id="system">
+ <h2>System metrics</h2>
+ <table id="system_metrics_table" class="table table-striped">
+ <thead>
+ <tr>
+ <th>CPU (%)</th>
+ <th>Load Average (<span id="proc-cores"></span> cores)</th>
+ <th>System Used RAM (%)</th>
+ <th>LLAP Open File #</th>
+ </tr>
+ </thead>
<tbody id="executors_body">
+ <tr>
+ <td> <span id="proc-cpu"> </span> <span id="proc-cpu-trend"> </span> </td>
+ <td> <span id="system-loadavg"> </span> <span id="system-loadavg-trend"> </span> </td>
+ <td> <span id="system-ram"> </span> <span id="system-ram-trend"> </span> </td>
+ <td> <span id="proc-fds"> </span> <span id="proc-fds-trend"> </span> </td>
+ </tr>
</tbody>
</table>
</section>
http://git-wip-us.apache.org/repos/asf/hive/blob/5c078940/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js b/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js
index 2ecdd4b..b67a6e3 100644
--- a/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js
+++ b/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js
@@ -16,97 +16,212 @@
* limitations under the License.
*/
-$(function() {
- // JMX json keys
- const heapUsedKey = "MemHeapUsedM";
- const heapMaxKey = "MemHeapMaxM";
-
- const cacheUsedKey = "CacheCapacityUsed";
- const cacheMaxKey = "CacheCapacityTotal";
- const cacheHitKey = "CacheHitRatio";
- const cacheRequestKey = "CacheReadRequests";
-
- const numExecutorsKey = "NumExecutors";
- const executorUsedKey = "ExecutorsStatus";
- const numQueuedKey = "ExecutorNumQueuedRequests";
- const preemptionTimeLostKey = "PreemptionTimeLost";
-
- function addRow(table, data, plot) {
- var row = $("<tr>");
- table.append(row);
- for (var i in data) {
- var item = $("<td>");
- item.append(data[i][data[i].length - 1]);
- row.append(item);
- if (plot[i]) {
- var line = $("<span>");
- line.css("padding-left", "0.5em");
- item.append(' ');
- item.append(line);
- line.sparkline(data[i]);
+/* llap (singletons) */
+var llap = {"model" : {}, "view" : {}}
+
+if (!("console" in window)) {
+ window.console = {"log" : function() {/*nothing*/}};
+}
+
+(
+/* anonymous closure to prevent variable leakage */
+function () {
+
+var trendlist = function(max) {
+ var list = [];
+ list.fill(0,0,max);
+ list.peek = function() {
+ if(this.length != 0) { return this[this.length-1] }
+ }
+ list.add = function(v) {
+ while (this.length < max) {
+ this.push(0);
}
- }
- }
-
- function createTable(table, data, plot) {
- table.empty();
- addRow(table, data, plot);
- }
-
- function addLineItem(list, newItem, max) {
- max = max || 30;
- list.push(newItem);
- if(list.length < max) {
- while(list.length < max) {
- list.push(0);
+ this.push(v);
+ while(this.length > max) {
+ this.shift();
}
- } else if (list.length > max) {
- while(list.length > max) {
- list.shift();
+ }
+ return list
+}
+
+var jmxbean = function(jmx, name) {
+ var beans = jmx["beans"].filter(function(b, i, a) { return b.name.match(name) });
+ return beans.pop();
+}
+
+
+llap.model.JvmMetrics = new function() {
+ this.name = "Hadoop:service=LlapDaemon,name=JvmMetrics";
+ this.heap_used = 0;
+ this.heap_max = 0;
+ this.heap_rate = trendlist(50);
+ this.gc_times = trendlist(50);
+ this.old_gc = 0;
+ this.push = function(jmx) {
+ var bean = jmxbean(jmx, this.name);
+ this.hostname = bean["tag.Hostname"];
+ this.heap_max = bean["MemHeapMaxM"];
+ this.heap_used = bean["MemHeapUsedM"];
+ this.heap_rate.add((this.heap_used*100.0)/this.heap_max);
+ var new_gc = (bean["GcTimeMillis"]/ 1000.0);
+ if (this.old_gc != 0) {
+ this.gc_times.add((new_gc - this.old_gc) / 1000.0);
}
- }
- }
+ this.old_gc = new_gc;
+ }
+ return this;
+}
- var heapUsed = [], heapMax = [], heapUseRate = [];
- var cacheUsed = [], cacheMax = [], cacheUseRate = [],
- cacheRequest = [], cacheHit = [];
- var executorsUsed = [], numExecutors = [], executorUseRate = [],
- queue = [], execAndQueue = [], preemptionTimeLost = [];
+llap.model.LlapDaemonCacheMetrics = new function() {
+ this.name = "Hadoop:service=LlapDaemon,name=LlapDaemonCacheMetrics";
+ this.hit_rate = trendlist(50);
+ this.fill_rate = trendlist(50);
+ this.push = function(jmx) {
+ var bean = jmxbean(jmx, this.name);
+ this.cache_max = bean["CacheCapacityTotal"]/(1024*1024);
+ this.cache_used = bean["CacheCapacityUsed"]/(1024*1024);
+ this.cache_reqs = bean["CacheReadRequests"];
+ this.fill_rate.add((this.cache_used*100.0)/this.cache_max);
+ this.hit_rate.add(bean["CacheHitRatio"]*100.0);
+ }
+ return this;
+}
- $("#hostname").append(location.hostname);
- setInterval(function() {
- $.getJSON("/jmx", function(jmx){
- var data = jmx["beans"];
- for (var i in data) {
- if (heapUsedKey in data[i]) addLineItem(heapUsed, data[i][heapUsedKey].toFixed(1));
- if (heapMaxKey in data[i]) addLineItem(heapMax, data[i][heapMaxKey].toFixed(1));
- if (cacheUsedKey in data[i]) addLineItem(cacheUsed, data[i][cacheUsedKey] / 1024 / 1024); // Exchange Byte to MB
- if (cacheMaxKey in data[i]) addLineItem(cacheMax, data[i][cacheMaxKey] / 1024 / 1024);
+llap.model.LlapDaemonInfo = new function() {
+ this.name = "Hadoop:service=LlapDaemon,name=LlapDaemonInfo";
+ this.active_rate = trendlist(50);
+ this.push = function(jmx) {
+ var bean = jmxbean(jmx, this.name);
+ this.executors = bean["NumExecutors"];
+ this.active = bean["ExecutorsStatus"];
+ this.active_rate.add(this.active.length);
+ }
+}
- if (cacheHitKey in data[i]) addLineItem(cacheHit, data[i][cacheHitKey] * 100);
- if (cacheRequestKey in data[i]) addLineItem(cacheRequest, data[i][cacheRequestKey]);
+llap.model.LlapDaemonExecutorMetrics = new function() {
+ this.name = "Hadoop:service=LlapDaemon,name=LlapDaemonExecutorMetrics";
+ this.queue_rate = trendlist(50);
+ this.push = function(jmx) {
+ var bean = jmxbean(jmx, this.name);
+ this.queue_rate.add(bean["ExecutorNumQueuedRequests"] || 0);
+ this.lost_time = bean["PreemptionTimeLost"];
+ this.num_tasks = bean["ExecutorTotalRequestsHandled"];
+ this.interrupted_tasks = bean["ExecutorTotalInterrupted"];
+ this.failed_tasks = bean["ExecutorTotalExecutionFailure"];
+ }
+ return this;
+}
- if (numExecutorsKey in data[i]) addLineItem(numExecutors, data[i][numExecutorsKey]);
- if (executorUsedKey in data[i]) addLineItem(executorsUsed, data[i][executorUsedKey].length);
- if (numQueuedKey in data[i]) addLineItem(queue, data[i][numQueuedKey]);
- if (preemptionTimeLostKey in data[i]) addLineItem(preemptionTimeLost, data[i][preemptionTimeLostKey]);
- }
- addLineItem(heapUseRate, (heapUsed[heapUsed.length - 1] / heapMax[heapMax.length - 1] * 100).toFixed(1));
- createTable($("#heap_body"),
- [heapUsed, heapMax, heapUseRate],
- [false, false, true]);
- addLineItem(cacheUseRate, (cacheUsed[cacheUsed.length - 1] / cacheMax[cacheMax.length - 1] * 100).toFixed(1));
- createTable($("#cache_body"),
- [cacheUsed, cacheMax, cacheUseRate, cacheRequest, cacheHit],
- [false, false, true, false, true]);
- addLineItem(executorUseRate,
- executorsUsed[executorsUsed.length - 1] / numExecutors[numExecutors.length - 1] * 100);
- addLineItem(execAndQueue, executorsUsed[executorsUsed.length - 1] + queue[queue.length - 1]);
- createTable($("#executors_body"),
- [executorsUsed, numExecutors, executorUseRate, queue, execAndQueue, preemptionTimeLost],
- [false, false, true, false, true, false]);
+llap.model.OperatingSystem = new function() {
+ this.name = "java.lang:type=OperatingSystem";
+ this.sys_cpu_rate = trendlist(50);
+ this.proc_cpu_rate = trendlist(50);
+ this.loadavg_rate = trendlist(50);
+ this.used_ram_rate = trendlist(50);
+ this.open_fds_rate = trendlist(50);
+ this.push = function(jmx) {
+ var bean = jmxbean(jmx, this.name);
+ this.sys_cpu_rate.add(Math.max(bean["SystemCpuLoad"]-bean["ProcessCpuLoad"],0)*100);
+ this.proc_cpu_rate.add(bean["ProcessCpuLoad"]*100);
+ this.loadavg_rate.add(bean["SystemLoadAverage"]);
+ this.used_ram_rate.add(100*(bean["TotalPhysicalMemorySize"] - bean["FreePhysicalMemorySize"])/bean["TotalPhysicalMemorySize"]);
+ this.open_fds_rate.add(bean["OpenFileDescriptorCount"]);
+ this.cpu_cores = bean["AvailableProcessors"];
+ }
+}
+
+llap.view.Hostname = new function () {
+ this.refresh = function() {
+ $("#hostname").text(llap.model.JvmMetrics.hostname);
+ }
+}
+
+llap.view.Heap = new function () {
+ this.refresh = function() {
+ var model = llap.model.JvmMetrics;
+ $("#heap-used").text(model.heap_used.toFixed(2));
+ $("#heap-max").text(model.heap_max.toFixed(2));
+ $("#heap-rate").text(model.heap_rate.peek().toFixed(2));
+ $("#heap-trend").sparkline(model.heap_rate);
+ $("#heap-gc").text(model.old_gc.toFixed(2));
+ $("#heap-gc-trend").sparkline(model.gc_times, {type: 'bar', barColor: 'red'});
+ }
+}
+
+llap.view.Cache = new function () {
+ this.refresh = function() {
+ var model = llap.model.LlapDaemonCacheMetrics;
+ $("#cache-used").text(model.cache_used.toFixed(2));
+ $("#cache-max").text(model.cache_max.toFixed(2));
+ $("#cache-fill-rate").text(model.fill_rate.peek().toFixed(2));
+ $("#cache-fill-trend").sparkline(model.fill_rate);
+ $("#cache-requests").text(model.cache_reqs.toFixed(0));
+ $("#cache-hits").text(model.hit_rate.peek().toFixed(2));
+ $("#cache-hits-trend").sparkline(model.hit_rate);
+ }
+}
+
+llap.view.Executors = new function () {
+ this.refresh = function() {
+ var model = llap.model.LlapDaemonInfo;
+ $("#executors-used").text(model.active.length);
+ $("#executors-max").text(model.executors);
+ $("#executors-rate").text(((model.active_rate.peek() * 100.0)/model.executors).toFixed(0));
+ $("#executors-trend").sparkline(model.active_rate);
+
+ var model1 = llap.model.LlapDaemonExecutorMetrics;
+ $("#executors-queue").text(model1.queue_rate.peek().toFixed(0));
+ $("#executors-pending").text((model1.queue_rate.peek() + model.active_rate.peek()).toFixed(0));
+ $("#executors-pending-trend").sparkline(model1.queue_rate.map(
+ function(v,i) { return v + model.active_rate[i] }
+ ));
+ }
+}
+
+llap.view.Tasks = new function() {
+ this.refresh = function() {
+ var model = llap.model.LlapDaemonExecutorMetrics;
+ $("#fragments-total").text(model.num_tasks);
+ $("#fragments-failed").text(model.failed_tasks);
+ $("#fragments-preempted").text(model.interrupted_tasks);
+ $("#fragments-preemption-time").text((model.lost_time / 1000.0).toFixed(3));
+ }
+}
+
+llap.view.System = new function() {
+ this.refresh = function() {
+ var model = llap.model.OperatingSystem;
+ $("#proc-cores").text(model.cpu_cores);
+ $("#proc-cpu").text(model.proc_cpu_rate.peek().toFixed(2));
+ $("#proc-cpu-trend").sparkline(model.proc_cpu_rate.map(
+ function(v, i) { return [model.sys_cpu_rate[i],v]; }
+ ), {type: 'bar', stackedBarColor: ['red','green']});
+
+ $("#system-loadavg").text(model.loadavg_rate.peek());
+ $("#system-loadavg-trend").sparkline(model.loadavg_rate);
+
+ $("#system-ram").text(model.used_ram_rate.peek().toFixed(2));
+ $("#system-ram-trend").sparkline(model.used_ram_rate);
+
+ $("#proc-fds").text(model.open_fds_rate.peek().toFixed(0));
+ $("#proc-fds-trend").sparkline(model.open_fds_rate);
+ }
+}
+
+})();
+
+$(function() {
+ var models = [llap.model.JvmMetrics, llap.model.LlapDaemonCacheMetrics, llap.model.LlapDaemonExecutorMetrics, llap.model.LlapDaemonInfo, llap.model.OperatingSystem]
+
+ var views = [llap.view.Hostname, llap.view.Heap, llap.view.Cache, llap.view.Executors, llap.view.Tasks, llap.view.System]
+
+ setInterval(function() {
+ $.getJSON("/jmx", function(jmx){
+ models.forEach(function (m) { m.push(jmx); });
+ views.forEach(function (v) { v.refresh(); });
});
}, 1000); // Update par sec
});