You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2024/02/12 19:58:08 UTC
(impala) 02/03: IMPALA-12688: Support JSON profile imports in webUI
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 47dd78d8381cadc99b65d347327e9dd9f2a6ae22
Author: Surya Hebbar <sh...@cloudera.com>
AuthorDate: Mon Jan 8 18:23:13 2024 +0530
IMPALA-12688: Support JSON profile imports in webUI
A button has been added to import query profiles on the queries page.
The button opens a system dialog to browse only "*.json" files from
client's local file system. This allows multiple JSON query profiles
to be imported.
JSON profiles are read from file system using FileReader API and then
these imported query profiles are stored in a NoSQL database using
browser's IndexedDB API.
If an error is encountered while parsing JSON profile, it is displayed
on the queries page.
IndexedDB is a large-scale NoSQL database system, that can be read
and written to asynchronously. Hence, the file size and number of JSON
profiles is only limited by the storage available on the client system.
The storage limit is browser dependent, in chromium based browsers
upto 80% of local storage may be used by IndexedDB.
The profile is parsed for the following attributes, which will be
displayed on the queries page. Number of imported queries is also
displayed.
- Query ID
- User
- Default Db
- Query Type
- Start Time
- End Time
- Bytes Read
- Bytes Sent
- Query State
- No. of rows fetched
- Resource Pool
- Query Statement
After importing or deleting query profiles, page will scroll back
to the imported query profiles section.
The queries are sorted based on descending order of the "Start Time".
A button is displayed beside the import button to easily clear all
the imported queries at once.
On opening any of the imported queries, its query timeline will be
displayed. Currently the following tabs are supported for imported
queries.
- Query Statement
- Query Timeline
- Query Text Plan
Syntax highlighting on query statement tab has also been supported.
Change-Id: Ife6eb59bf2030fd19fc92aaf134eb51c609e04d0
Reviewed-on: http://gerrit.cloudera.org:8080/20867
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
www/queries.tmpl | 191 ++++++++++++++++++++-
www/query_plan_text.tmpl | 42 ++++-
www/query_stmt.tmpl | 49 +++++-
www/query_timeline.tmpl | 92 +++++++---
.../query_timeline/fragment_metrics_diagram.js | 14 +-
.../query_timeline/host_utilization_diagram.js | 11 +-
6 files changed, 363 insertions(+), 36 deletions(-)
diff --git a/www/queries.tmpl b/www/queries.tmpl
index 95a763a2f..fc3097f65 100644
--- a/www/queries.tmpl
+++ b/www/queries.tmpl
@@ -17,9 +17,17 @@ specific language governing permissions and limitations
under the License.
-->
{{> www/common-header.tmpl }}
+
+<style>
+#json_profile_chooser {
+ display: none;
+}
+</style>
+
<h2>Queries</h2>
-<p class="lead">This page lists all running queries, plus any completed queries that are
-archived in memory. The size of that archive is controlled with the
+<p class="lead">This page lists all running queries plus completed queries
+that are archived in memory, and imported query profiles.
+The size of that archive is controlled with the
<samp>--query_log_size</samp> command line parameter.</p>
<p>The length of the statements are controlled with the <samp>--query_stmt_size</samp>
command line parameter.</p>
@@ -205,6 +213,37 @@ command line parameter.</p>
{{/completed_queries}}
</table>
+<h3 id="imported_queries_header">
+ 0 Imported Query Profiles
+ <sup><a href='#' data-toggle="tooltip" title="These are locally stored queries parsed from JSON query profiles.">[?]</a></sup>
+ <input id="json_profile_chooser" type="file" accept=".json" onchange="uploadProfile();" multiple/>
+ <label for="json_profile_chooser">
+ <span class="btn btn-primary">Import JSON Profile</span>
+ </label>
+ <input id="clear_profiles_button" type="button" class="btn btn-primary" title="Clear All" onclick="clearProfiles();" value="X"/>
+</h3>
+
+<table id="imported_queries_table" class='table table-hover table-border'>
+ <tr>
+ <th title="{{tips_query_id}}">Query ID</th>
+ <th title="{{tips_user}}">User</th>
+ <th title="{{tips_default_db}}">Default Db</th>
+ <th title="{{tips_query_type}}">Query Type</th>
+ <th title="{{tips_start_time}}">Start Time</th>
+ <th title="{{tips_end_time}}">End Time</th>
+ <th>
+ <span title="{{tips_bytes_read}}">Bytes Read</span>
+ <hr style="margin-top:0px;margin-bottom:0px;"/>
+ <span title="{{tips_bytes_sent}}">Bytes Sent</span>
+ </th>
+ <th title="{{tips_state}}">State</th>
+ <th title="{{tips_rows_fetched}}"># rows fetched</th>
+ <th title="{{tips_resource_pool}}">Resource Pool</th>
+ <th title="{{tips_statement}}">Statement</th>
+ <th>Delete</th>
+ </tr>
+</table>
+
<h3>Query Locations</h3>
<table class='table table-hover table-bordered'>
@@ -221,5 +260,153 @@ command line parameter.</p>
</tr>
{{/query_locations}}
</table>
+<script>
+ var dbOpenReq = indexedDB.open("imported_queries");
+ var db;
+
+ function insertRowVal(row, val) {
+ row.insertCell().innerHTML = val;
+ }
+
+ function setScrollReload() {
+ localStorage.setItem("imported", "T");
+ window.location.reload();
+ }
+
+ function deleteProfile(el) {
+ var row = el.parentElement.parentElement;
+ var profileStore = db.transaction("profiles", "readwrite").objectStore("profiles");
+ profileStore.delete(row.cells[0].textContent).onsuccess = () => {
+ setScrollReload();
+ };
+ }
+
+ function clearProfiles() {
+ db.transaction("profiles", "readwrite").objectStore("profiles").clear();
+ setScrollReload();
+ }
+
+ function uploadProfile() {
+ var uploadCount = 0;
+ for (var i = 0; i < json_profile_chooser.files.length; i++) {
+ var fileReader = new FileReader();
+ fileReader.readAsText(json_profile_chooser.files[i]);
+ fileReader.onload = (e) => {
+ try {
+ var profile = JSON.parse(e.target.result).contents;
+ var val = profile.profile_name;
+ var query = {};
+ query.id = val.substring(val.indexOf("=") + 1, val.length - 1);
+ query.user = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "User").value;
+ query.default_db = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Default Db").value;
+ query.type = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Query Type").value;
+ query.start_time = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Start Time").value;
+ query.end_time = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "End Time").value;
+ query.bytes_read = profile.child_profiles[2].counters
+ .find(({counter_name}) => counter_name === "TotalBytesRead").value;
+ query.bytes_read = getReadableSize(query.bytes_read, 2);
+ query.bytes_sent = profile.child_profiles[2].counters
+ .find(({counter_name}) => counter_name === "TotalBytesSent").value;
+ query.bytes_sent = getReadableSize(query.bytes_sent, 2);
+ query.state = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Query State").value;
+ query.rows_fetched = profile.child_profiles[1].counters
+ .find(({counter_name}) => counter_name === "NumRowsFetched").value;
+ query.resource_pool = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Request Pool").value;
+ query.statement = profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Sql Statement").value;
+ if (query.statement.length > 250) {
+ query.statement = query.statement.substring(0, 250) + "...";
+ }
+ query.profile = profile;
+
+ var profileStore = db.transaction("profiles", "readwrite").objectStore("profiles");
+ profileStore.put(query).onsuccess = () => {
+ uploadCount++;
+ if (uploadCount == json_profile_chooser.files.length) {
+ setScrollReload();
+ }
+ };
+ } catch (err) {
+ var alertMessage = document.createElement("span");
+ alertMessage.className = "alert-sm alert-danger";
+ alertMessage.textContent = "Error parsing some JSON profiles";
+ setTimeout(setScrollReload, 1500);
+ imported_queries_header.appendChild(alertMessage);
+ console.log(err);
+ }
+ };
+ }
+ }
+
+ dbOpenReq.onupgradeneeded = (e) => {
+ db = e.target.result;
+ var profileStore = db.createObjectStore("profiles", { keyPath : "id" });
+
+ profileStore.createIndex("start_time", "start_time");
+ };
+
+ dbOpenReq.onsuccess = (e) => {
+ db = e.target.result;
+ db.onerror = (e) => {
+ console.log("IndexedDB error");
+ console.log(e);
+ }
+
+ var profileStore = db.transaction("profiles", "readonly").objectStore("profiles");
+ var countReq = profileStore.count();
+
+ countReq.onsuccess = () => {
+ if (countReq.result) {
+ imported_queries_header.innerHTML = imported_queries_header.innerHTML
+ .replace("0", String(countReq.result));
+ } else {
+ clear_profiles_button.remove();
+ }
+ };
+
+ profileStore.index("start_time").openCursor(null, "prev").onsuccess = (e) => {
+ var cursor = e.target.result;
+ if (cursor) {
+ var query = cursor.value;
+ var row = imported_queries_table.insertRow();
+ var query_link = document.createElement("a");
+ query_link.href = `/query_timeline?query_id=${query.id}&imported=true`;
+ query_link.innerHTML = query.id;
+ insertRowVal(row, query_link.outerHTML);
+ insertRowVal(row, query.user);
+ insertRowVal(row, query.default_db);
+ insertRowVal(row, query.type);
+ insertRowVal(row, query.start_time);
+ insertRowVal(row, query.end_time);
+ insertRowVal(row, `${query.bytes_read}
+ <hr style="margin-top:0px;margin-bottom:0px;"/>${query.bytes_sent}`);
+ insertRowVal(row, query.state);
+ insertRowVal(row, query.rows_fetched);
+ insertRowVal(row, query.resource_pool);
+ insertRowVal(row, query.statement);
+ var deleteButton = document.createElement("input");
+ deleteButton.type = "button";
+ deleteButton.value = "X";
+ deleteButton.className = "btn btn-primary";
+ deleteButton.setAttribute("onclick", "deleteProfile(this);");
+ insertRowVal(row, deleteButton.outerHTML);
+ cursor.continue();
+ }
+ };
+ };
+ window.onload = () => {
+ if (localStorage.getItem("imported") == "T") {
+ imported_queries_header.scrollIntoView({ behavior : "instant", block : "start" });
+ localStorage.setItem("imported", "F");
+ }
+ };
+</script>
{{> www/common-footer.tmpl }}
diff --git a/www/query_plan_text.tmpl b/www/query_plan_text.tmpl
index 3f922f0fe..add1e4448 100644
--- a/www/query_plan_text.tmpl
+++ b/www/query_plan_text.tmpl
@@ -21,10 +21,50 @@ under the License.
{{> www/query_detail_tabs.tmpl }}
-<pre>{{plan}}</pre>
+<pre id="query_plan">{{plan}}</pre>
<script>
$("#plan-text-tab").addClass("active");
+
+
+var dbOpenReq = indexedDB.open("imported_queries");
+var db;
+
+var supported_tabs = ["Query", "Timeline", "Text plan"];
+
+if (window.location.search.includes("imported")) {
+ var alertMessage = document.getElementsByClassName("alert alert-danger")[0];
+ if (alertMessage) {
+ alertMessage.remove();
+ }
+ var nav_links = document.getElementsByClassName("nav nav-tabs")[0];
+ nav_links = nav_links.getElementsByClassName("nav-link");
+ for (var i = 0; i < nav_links.length;) {
+ if (supported_tabs.includes(nav_links[i].textContent)) {
+ nav_links[i].href = `${nav_links[i].href}&imported=true`;
+ i++;
+ } else {
+ nav_links[i].parentElement.remove();
+ }
+ }
+
+ dbOpenReq.onsuccess = (e) => {
+ db = e.target.result;
+ db.onerror = (e) => {
+ console.log("IndexedDB error");
+ console.log(e);
+ }
+ var profileStore = db.transaction("profiles", "readonly").objectStore("profiles");
+ var query = {};
+ query.id = window.location.search;
+ query.id = query.id.substring(10, query.id.indexOf("&"));
+ profileStore.get(query.id).onsuccess = (e) => {
+ query = e.target.result;
+ query_plan.textContent = query.profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Plan").value;
+ };
+ };
+}
</script>
{{> www/common-footer.tmpl }}
diff --git a/www/query_stmt.tmpl b/www/query_stmt.tmpl
index a5886ff56..188b40c89 100644
--- a/www/query_stmt.tmpl
+++ b/www/query_stmt.tmpl
@@ -30,10 +30,57 @@ under the License.
<script src="{{ __common__.host-url }}/www/highlight/highlight.pack.js"></script>
<script>hljs.initHighlightingOnLoad();</script>
-{{?stmt}}<pre class="code"><code>{{stmt}}</code></pre>{{/stmt}}
+<span id="tab_body">{{?stmt}}<pre class="code"><code>{{stmt}}</code></pre>{{/stmt}}</span>
<script>
$("#stmt-tab").addClass("active");
+
+var dbOpenReq = indexedDB.open("imported_queries");
+var db;
+
+var supported_tabs = ["Query", "Timeline", "Text plan"];
+
+if (window.location.search.includes("imported")) {
+ var alertMessage = document.getElementsByClassName("alert alert-danger")[0];
+ if (alertMessage) {
+ alertMessage.remove();
+ }
+ var nav_links = document.getElementsByClassName("nav nav-tabs")[0];
+ nav_links = nav_links.getElementsByClassName("nav-link");
+ for (var i = 0; i < nav_links.length;) {
+ if (supported_tabs.includes(nav_links[i].textContent)) {
+ nav_links[i].href = `${nav_links[i].href}&imported=true`;
+ i++;
+ } else {
+ nav_links[i].parentElement.remove();
+ }
+ }
+
+ dbOpenReq.onsuccess = (e) => {
+ db = e.target.result;
+ db.onerror = (e) => {
+ console.log("IndexedDB error");
+ console.log(e);
+ }
+ var profileStore = db.transaction("profiles", "readonly").objectStore("profiles");
+ var query = {};
+ query.id = window.location.search;
+ query.id = query.id.substring(10, query.id.indexOf("&"));
+ profileStore.get(query.id).onsuccess = (e) => {
+ query = e.target.result;
+ var sql_query = query.profile.child_profiles[0].info_strings
+ .find(({key}) => key === "Sql Statement").value;
+ var sql_stmt_body = document.createElement("pre");
+ sql_stmt_body.className = "code";
+ var sql_code = sql_stmt_body.appendChild(document.createElement("code"))
+ sql_code.innerHTML = sql_query;
+ tab_body.innerHTML = "";
+ tab_body.appendChild(sql_stmt_body);
+ hljs.highlightBlock(sql_stmt_body);
+ };
+ };
+}
+
</script>
{{> www/common-footer.tmpl }}
diff --git a/www/query_timeline.tmpl b/www/query_timeline.tmpl
index 4de641c69..8288e5349 100644
--- a/www/query_timeline.tmpl
+++ b/www/query_timeline.tmpl
@@ -162,37 +162,83 @@ $("#plan-timing-tab").addClass("active");
import {renderTimingDiagram, collectFragmentEventsFromProfile, ntics, set_ntics} from
"./www/scripts/query_timeline/fragment_diagram.js";
-import {collectUtilizationFromProfile, toogleUtilizationVisibility} from
- "/www/scripts/query_timeline/host_utilization_diagram.js";
-import {collectFragmentMetricsFromProfile} from
+import {collectUtilizationFromProfile, toogleUtilizationVisibility,
+ destroyUtilizationChart}
+ from "/www/scripts/query_timeline/host_utilization_diagram.js";
+import {collectFragmentMetricsFromProfile, closeFragmentMetricsChart} from
"/www/scripts/query_timeline/fragment_metrics_diagram.js";
-import {profile, set_profile, diagram_width, set_diagram_width, border_stroke_width,
- resizeHorizontalAll, maxts} from "/www/scripts/query_timeline/global_members.js";
+import {profile, set_profile, maxts, set_maxts, diagram_width, set_diagram_width,
+ border_stroke_width, resizeHorizontalAll}
+ from "/www/scripts/query_timeline/global_members.js";
var chart_export_style;
var last_maxts;
-window.onload = function refresh() {
- var req = new XMLHttpRequest();
- req.onload = function() {
- if (req.status == 200) {
- set_profile(JSON.parse(req.responseText)["profile_json"]);
- collectFragmentEventsFromProfile();
- collectUtilizationFromProfile();
- collectFragmentMetricsFromProfile();
- if (last_maxts != maxts) {
- renderTimingDiagram();
- last_maxts = maxts;
- }
- toogleUtilizationVisibility();
- if (profile.child_profiles[0].info_strings.find(({key}) =>
- key === "End Time").value == "") {
- setTimeout(refresh, 1000);
+var dbOpenReq = indexedDB.open("imported_queries");
+var db;
+
+var supported_tabs = ["Query", "Timeline", "Text plan"];
+
+function refreshView() {
+ collectFragmentEventsFromProfile();
+ collectUtilizationFromProfile();
+ collectFragmentMetricsFromProfile();
+ if (last_maxts != maxts) {
+ renderTimingDiagram();
+ last_maxts = maxts;
+ }
+ toogleUtilizationVisibility();
+}
+
+
+if (window.location.search.includes("imported")) {
+ var alertMessage = document.getElementsByClassName("alert alert-danger")[0];
+ if (alertMessage) {
+ alertMessage.remove();
+ }
+ var nav_links = document.getElementsByClassName("nav nav-tabs")[0];
+ nav_links = nav_links.getElementsByClassName("nav-link");
+ for (var i = 0; i < nav_links.length;) {
+ if (supported_tabs.includes(nav_links[i].textContent)) {
+ nav_links[i].href = `${nav_links[i].href}&imported=true`;
+ i++;
+ } else {
+ nav_links[i].parentElement.remove();
+ }
+ }
+
+ dbOpenReq.onsuccess = (e) => {
+ db = e.target.result;
+ db.onerror = (e) => {
+ console.log("IndexedDB error");
+ console.log(e);
+ }
+ var profileStore = db.transaction("profiles", "readonly").objectStore("profiles");
+ var query = {};
+ query.id = window.location.search;
+ query.id = query.id.substring(10, query.id.indexOf("&"));
+ profileStore.get(query.id).onsuccess = (e) => {
+ query = e.target.result;
+ set_profile(query.profile);
+ refreshView();
+ };
+ };
+} else {
+ window.onload = function refreshProfile() {
+ var req = new XMLHttpRequest();
+ req.onload = function() {
+ if (req.status == 200) {
+ set_profile(JSON.parse(req.responseText)["profile_json"]);
+ refreshView();
+ if (profile.child_profiles[0].info_strings.find(({key}) =>
+ key === "End Time").value == "") {
+ setTimeout(refreshProfile, 1000);
+ }
}
}
+ req.open("GET", make_url("/query_timeline?query_id={{query_id}}&json"), true);
+ req.send();
}
- req.open("GET", make_url("/query_timeline?query_id={{query_id}}&json"), true);
- req.send();
}
window.addEventListener('mousemove', function(e) {
diff --git a/www/scripts/query_timeline/fragment_metrics_diagram.js b/www/scripts/query_timeline/fragment_metrics_diagram.js
index 54a0a8c68..bb6af59f2 100644
--- a/www/scripts/query_timeline/fragment_metrics_diagram.js
+++ b/www/scripts/query_timeline/fragment_metrics_diagram.js
@@ -283,6 +283,13 @@ export function updateFragmentMetricsChartOnClick(click_event) {
toogleFragmentMetricsVisibility();
}
+export function closeFragmentMetricsChart() {
+ fragment_id_selections.clear();
+ fragment_metrics_chart = destroyChart(fragment_metrics_chart, fragment_metrics_diagram);
+ toogleFragmentMetricsVisibility();
+ setTimingDiagramDimensions();
+}
+
fragment_metrics_resize_bar.addEventListener('mousedown',
function dragResizeBarBegin(mousedown_e) {
fragment_metrics_resize_bar.removeEventListener('mousedown', dragResizeBarBegin);
@@ -294,12 +301,7 @@ fragment_metrics_resize_bar.addEventListener('mousedown',
});
});
-fragment_metrics_close_btn.addEventListener('click', function() {
- fragment_id_selections.clear();
- fragment_metrics_chart = destroyChart(fragment_metrics_chart, fragment_metrics_diagram);
- toogleFragmentMetricsVisibility();
- setTimingDiagramDimensions();
-});
+fragment_metrics_close_btn.addEventListener('click', closeFragmentMetricsChart);
fragment_metrics_close_btn.style.height = `${diagram_controls_height}px`;
fragment_metrics_close_btn.style.fontSize = `${diagram_controls_height / 2}px`;
diff --git a/www/scripts/query_timeline/host_utilization_diagram.js b/www/scripts/query_timeline/host_utilization_diagram.js
index 8ef4b69b0..bffd4987e 100644
--- a/www/scripts/query_timeline/host_utilization_diagram.js
+++ b/www/scripts/query_timeline/host_utilization_diagram.js
@@ -134,6 +134,7 @@ function initializeUtilizationChart() {
unload : true
});
var chart_width = diagram_width - margin_chart_end - name_width;
+ prev_utilization_num_samples = 0;
host_utilization_resize_bar.style.marginLeft = `${name_width + chart_width / 4}px`;
host_utilization_resize_bar.style.width = `${chart_width / 2}px`;
host_utilization_resize_bar.style.marginRight = `${chart_width / 4}px`;
@@ -302,6 +303,12 @@ export function collectUtilizationFromProfile() {
}
}
+export function destroyUtilizationChart() {
+ host_utilization_chart = destroyChart(host_utilization_chart, host_utilization_diagram);
+ toogleUtilizationVisibility();
+ setTimingDiagramDimensions();
+}
+
host_utilization_resize_bar.addEventListener('mousedown',
function dragResizeBarBegin(mousedown_e) {
host_utilization_resize_bar.removeEventListener('mousedown', dragResizeBarBegin);
@@ -316,9 +323,7 @@ host_utilization_resize_bar.addEventListener('mousedown',
host_utilization_close_btn.addEventListener('click', function(e) {
host_utilization_visible = false;
- host_utilization_chart = destroyChart(host_utilization_chart, host_utilization_diagram);
- toogleUtilizationVisibility();
- setTimingDiagramDimensions();
+ destroyUtilizationChart();
});
host_utilization_close_btn.style.height = `${diagram_controls_height}px`;