You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2023/04/24 21:40:37 UTC
[impala] branch master updated (2750ce59d -> 422006a80)
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 2750ce59d IMPALA-12073: Avoid catalogd exceptions caused by topic_update_tbl_max_wait_time_ms being set to 1
new 970c60096 IMPALA-11970: Optimized rendering for the query timeline display
new 422006a80 IMPALA-11950: Planner change for Iceberg metadata querying
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../catalog/iceberg/IcebergMetadataTable.java | 20 +-
.../apache/impala/planner/DistributedPlanner.java | 10 +
.../impala/planner/IcebergMetadataScanNode.java | 78 ++++
.../java/org/apache/impala/planner/ScanNode.java | 6 +-
.../apache/impala/planner/SingleNodePlanner.java | 13 +-
.../java/org/apache/impala/service/Frontend.java | 14 +-
.../org/apache/impala/planner/PlannerTest.java | 9 +
.../PlannerTest/iceberg-metadata-table-scan.test | 114 ++++++
www/query_timeline.tmpl | 414 +++++++++++----------
9 files changed, 461 insertions(+), 217 deletions(-)
create mode 100644 fe/src/main/java/org/apache/impala/planner/IcebergMetadataScanNode.java
create mode 100644 testdata/workloads/functional-planner/queries/PlannerTest/iceberg-metadata-table-scan.test
[impala] 01/02: IMPALA-11970: Optimized rendering for the query timeline display
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 970c6009600dd7b47fac52d391923aded0403bfc
Author: Surya Hebbar <sh...@cloudera.com>
AuthorDate: Tue Apr 11 00:23:28 2023 +0530
IMPALA-11970: Optimized rendering for the query timeline display
Decoupled the retrieval of attributes and rendering
to optimize window resize events for large JSON
query profiles in the timeline display.
A reload will be triggered after every second,
in case the query execution has not completed,
or due to errors in parsing the profile;
only valid profiles are rendered.
Replaces 'maximum()' and 'minimum()' functions with
'Math.max.apply()' and 'Math.min.apply()'.
Change-Id: Iec458602266863b6e313f52b26a2dc35cff0db0f
Reviewed-on: http://gerrit.cloudera.org:8080/19744
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
www/query_timeline.tmpl | 414 ++++++++++++++++++++++++------------------------
1 file changed, 210 insertions(+), 204 deletions(-)
diff --git a/www/query_timeline.tmpl b/www/query_timeline.tmpl
index f32a1bda0..18e84e5c7 100644
--- a/www/query_timeline.tmpl
+++ b/www/query_timeline.tmpl
@@ -62,6 +62,7 @@ under the License.
$("#plan-timing-tab").addClass("active");
+var profile = {};
var phases = [
{ color: "#C0C0FF", label: "Prepare" }
, { color: "#E0E0E0", label: "Open" }
@@ -69,29 +70,18 @@ var phases = [
, { color: "#C0FFFF", label: "Send First Batch" }
, { color: "#C0FFC0", label: "Process Remaining Batches" }
, { color: "#FFC0C0", label: "Close" }
-]
-
+];
var fragment_colors = ["#A9A9A9", "#FF8C00", "#8A2BE2", "#A52A2A", "#00008B", "#006400",
- "#228B22", "#4B0082", "#DAA520", "#008B8B", "#000000", "#DC143C"]
-
-var query_finished = false;
-var intervalId = 0;
-
-function minimum(numlist) {
- var minval = numlist[0];
- for (var i = 1; i < numlist.length; ++i) {
- minval=Math.min(minval, numlist[i]);
- }
- return minval;
-}
+ "#228B22", "#4B0082", "#DAA520", "#008B8B", "#000000", "#DC143C"];
+var fragments = [];
+var rownum = 0;
+var char_width = 6;
+var max_namelen = 0;
+var all_nodes = [];
+var receiver_nodes = [];
+var profile_available = false;
+var maxts = 0;
-function maximum(numlist) {
- var maxval = numlist[0];
- for (var i = 1; i < numlist.length; ++i) {
- maxval=Math.max(maxval, numlist[i]);
- }
- return maxval;
-}
function DrawBars(ctx, rownum, height, events, xoffset, px_per_ns) {
var color_idx = 0;
@@ -103,7 +93,7 @@ function DrawBars(ctx, rownum, height, events, xoffset, px_per_ns) {
var x = last_end;
var y = rownum * height;
- var endts = maximum(ev.tslist);
+ var endts = Math.max.apply(null, ev.tslist);
var width = xoffset + endts * px_per_ns - last_end;
last_end = x + width;
@@ -133,186 +123,186 @@ function DrawBars(ctx, rownum, height, events, xoffset, px_per_ns) {
});
}
-function renderTiming(ignored_arg) {
- if (req.status != 200) {
- return;
- }
- var json = JSON.parse(req.responseText);
- var profile = json["profile_json"];
- var qs = profile.child_profiles[0].info_strings.find(({ key }) => key === "Query State").value;
- if (qs == "FINISHED" ) {
- query_finished = true;
- }
-
-
- var header_canvas = document.getElementById('header_canvas');
- var timing_canvas = document.getElementById('timing_canvas');
- var footer_canvas = document.getElementById('footer_canvas');
-
- timing_canvas.width = window.innerWidth - 50;
- header_canvas.width = footer_canvas.width = timing_canvas.clientWidth;
-
- header_canvas.height = 15; // Height gets resized later
- timing_canvas.height = 200; // Height gets resized later
- footer_canvas.height = 15; // Height gets resized later
-
- var header_ctx = header_canvas.getContext('2d');
- var timing_ctx = timing_canvas.getContext('2d');
- var footer_ctx = footer_canvas.getContext('2d');
-
- var plan_order = document.getElementById("plan_order").checked;
-
- var rownum = 0;
- var maxts = 0;
- var max_namelen = 0;
- var char_width = 6;
+function collectFromProfile(ignored_arg) {
+ rownum = 0;
+ maxts = 0;
+ max_namelen = 0;
+ fragments = [];
var color_idx = 0;
- var fragments = [];
-
- var all_nodes = [];
- var receiver_nodes = [];
- // First pass: compute sizes
- profile.child_profiles[2].child_profiles.forEach(function(fp) {
-
- var cp = fp.child_profiles[0];
-
- if (cp.event_sequences != undefined) {
- var fevents = fp.child_profiles[0].event_sequences[0].events;
-
- // Build list of timestamps that spans instances for each event
- for (var en = 0; en < fevents.length; ++en) {
- fevents[en].tslist = [ fevents[en].timestamp ];
- for (var instance = 1; instance < fp.child_profiles.length; ++instance) {
- if (fp.child_profiles[instance].event_sequences != undefined) {
- fevents[en].tslist.push(fp.child_profiles[instance].event_sequences[0].events[en].timestamp);
+ all_nodes = [];
+ receiver_nodes = [];
+ try {
+ // First pass: compute sizes
+ profile.child_profiles[2].child_profiles.forEach(function(fp) {
+
+ var cp = fp.child_profiles[0];
+
+ if (cp.event_sequences != undefined) {
+ var fevents = fp.child_profiles[0].event_sequences[0].events;
+
+ // Build list of timestamps that spans instances for each event
+ for (var en = 0; en < fevents.length; ++en) {
+ fevents[en].tslist = [ fevents[en].timestamp ];
+ for (var instance = 1; instance < fp.child_profiles.length; ++instance) {
+ if (fp.child_profiles[instance].event_sequences != undefined) {
+ fevents[en].tslist.push(
+ fp.child_profiles[instance].event_sequences[0].events[en].timestamp);
+ }
}
}
- }
- fragment = {
- name: fp.profile_name,
- events: fevents,
- nodes: [ ],
- color: fragment_colors[color_idx]
- }
-
- // Pick a new color for each plan fragment
- color_idx = (color_idx + 1) % fragment_colors.length;
- maxts = Math.max(maxts, fevents[fevents.length - 1].timestamp);
- max_namelen = Math.max(max_namelen, fp.profile_name.length);
- var node_path = [];
- var node_stack = [];
- cp.child_profiles.forEach(function get_plan_nodes(pp, index) {
- if (pp.node_metadata != undefined) {
- node_path.push(index);
- var name_flds = pp.profile_name.split(/[()]/);
- var node_type = name_flds[0].trim();
- var node_id = name_flds.length > 1 ? name_flds[1].split(/[=]/)[1] : 0;
- node_name = pp.profile_name.replace("_NODE", "").replace("_"," ").replace("KrpcDataStreamSender", "SENDER").replace("Hash Join Builder", "JOIN BUILD").replace("join node_","")
- if (node_type.indexOf("SCAN_NODE") >= 0) {
- var table_name = pp.info_strings.find(({ key }) => key === "Table Name").value.split(/[.]/);
- node_name = node_name.replace("SCAN", "SCAN [" + table_name[table_name.length - 1] + "]");
- }
+ fragment = {
+ name: fp.profile_name,
+ events: fevents,
+ nodes: [ ],
+ color: fragment_colors[color_idx]
+ }
+ // Pick a new color for each plan fragment
+ color_idx = (color_idx + 1) % fragment_colors.length;
+ maxts = Math.max(maxts, fevents[fevents.length - 1].timestamp);
+ max_namelen = Math.max(max_namelen, fp.profile_name.length);
+ var node_path = [];
+ var node_stack = [];
+ cp.child_profiles.forEach(function get_plan_nodes(pp, index) {
+ if (pp.node_metadata != undefined) {
+ node_path.push(index);
+ var name_flds = pp.profile_name.split(/[()]/);
+ var node_type = name_flds[0].trim();
+ var node_id = name_flds.length > 1 ? name_flds[1].split(/[=]/)[1] : 0;
+ node_name = pp.profile_name.replace("_NODE", "").replace("_", " ")
+ .replace("KrpcDataStreamSender", "SENDER").replace("Hash Join Builder", "JOIN BUILD")
+ .replace("join node_", "");
+ if (node_type.indexOf("SCAN_NODE") >= 0) {
+ var table_name = pp.info_strings.find(({ key }) => key === "Table Name")
+ .value.split(/[.]/);
+ node_name = node_name.replace("SCAN",
+ "SCAN [" + table_name[table_name.length - 1] + "]");
+ }
- var is_receiver = node_type == "EXCHANGE_NODE" ||
- (node_type == "HASH_JOIN_NODE" && pp.num_children < 3);
-
- var is_sender = (node_type == "Hash Join Builder" ||
- node_type == "KrpcDataStreamSender");
- var parent_node;
- if (node_type == "PLAN_ROOT_SINK") {
- parent_node = undefined;
- } else if (pp.node_metadata.data_sink_id != undefined) {
- parent_node = receiver_nodes[node_id]; // Exchange sender dst
- } else if (pp.node_metadata.join_build_id != undefined) {
- parent_node = receiver_nodes[node_id]; // Join sender dst
- } else if (node_stack.length > 0) {
- parent_node = node_stack[node_stack.length - 1];
- } else if (all_nodes.length) {
- parent_node = all_nodes[all_nodes.length - 1];
- }
+ var is_receiver = node_type == "EXCHANGE_NODE" ||
+ (node_type == "HASH_JOIN_NODE" && pp.num_children < 3);
+
+ var is_sender = (node_type == "Hash Join Builder" ||
+ node_type == "KrpcDataStreamSender");
+ var parent_node;
+ if (node_type == "PLAN_ROOT_SINK") {
+ parent_node = undefined;
+ } else if (pp.node_metadata.data_sink_id != undefined) {
+ parent_node = receiver_nodes[node_id]; // Exchange sender dst
+ } else if (pp.node_metadata.join_build_id != undefined) {
+ parent_node = receiver_nodes[node_id]; // Join sender dst
+ } else if (node_stack.length > 0) {
+ parent_node = node_stack[node_stack.length - 1];
+ } else if (all_nodes.length) {
+ parent_node = all_nodes[all_nodes.length - 1];
+ }
- max_namelen = Math.max(max_namelen, node_name.length + node_stack.length + 1);
+ max_namelen = Math.max(max_namelen, node_name.length + node_stack.length + 1);
- if (pp.event_sequences != undefined) {
- var node_events = pp.event_sequences[0].events;
+ if (pp.event_sequences != undefined) {
+ var node_events = pp.event_sequences[0].events;
- // Start the instance event list for each event with timestamps from this instance
- for (var en = 0; en < node_events.length; ++en) {
- node_events[en].tslist = [ node_events[en].timestamp ];
- if (node_type == "HASH_JOIN_NODE" && (en == 1 || en == 2)) {
- node_events[en].no_bar = true;
+ // Start the instance event list for each event with timestamps from this instance
+ for (var en = 0; en < node_events.length; ++en) {
+ node_events[en].tslist = [ node_events[en].timestamp ];
+ if (node_type == "HASH_JOIN_NODE" && (en == 1 || en == 2)) {
+ node_events[en].no_bar = true;
+ }
}
}
- }
- var node = {
- name: node_name,
- type: node_type,
- node_id: node_id,
- num_children: 0,
- child_index: 0,
- metadata: pp.node_metadata,
- parent_node: parent_node,
- events: node_events,
- path: node_path.slice(0),
- is_receiver: is_receiver,
- is_sender: is_sender
- }
+ var node = {
+ name: node_name,
+ type: node_type,
+ node_id: node_id,
+ num_children: 0,
+ child_index: 0,
+ metadata: pp.node_metadata,
+ parent_node: parent_node,
+ events: node_events,
+ path: node_path.slice(0),
+ is_receiver: is_receiver,
+ is_sender: is_sender
+ }
- if (is_sender) {
- node.parent_node.sender_frag_index = fragments.length;
- }
+ if (is_sender) {
+ node.parent_node.sender_frag_index = fragments.length;
+ }
- if (is_receiver) {
- receiver_nodes[node_id] = node;
- }
+ if (is_receiver) {
+ receiver_nodes[node_id] = node;
+ }
- if (parent_node != undefined) {
- node.child_index = parent_node.num_children++;
- }
+ if (parent_node != undefined) {
+ node.child_index = parent_node.num_children++;
+ }
- all_nodes.push(node);
+ all_nodes.push(node);
- fragment.nodes.push(node);
+ fragment.nodes.push(node);
- if (pp.child_profiles != undefined) {
- node_stack.push(node);
- pp.child_profiles.forEach(get_plan_nodes);
- node = node_stack.pop();
+ if (pp.child_profiles != undefined) {
+ node_stack.push(node);
+ pp.child_profiles.forEach(get_plan_nodes);
+ node = node_stack.pop();
+ }
+ rownum++;
+ node_path.pop();
}
- rownum++;
- node_path.pop();
- }
- });
+ });
- // For each node, retrieve the instance timestamps for the remaining instances
- for (var ni = 0; ni < fragment.nodes.length; ++ni) {
- var node = fragment.nodes[ni];
- for (var cpn = 1; cpn < fp.child_profiles.length; ++cpn) {
- var cp = fp.child_profiles[cpn];
+ // For each node, retrieve the instance timestamps for the remaining instances
+ for (var ni = 0; ni < fragment.nodes.length; ++ni) {
+ var node = fragment.nodes[ni];
+ for (var cpn = 1; cpn < fp.child_profiles.length; ++cpn) {
+ var cp = fp.child_profiles[cpn];
- // Use the saved node path to traverse to the same position in this instance
- for (var pi = 0; pi < node.path.length; ++pi) {
- cp = cp.child_profiles[node.path[pi]];
- }
- console.assert(cp.node_metadata.data_sink_id == undefined ||
- cp.profile_name.indexOf("(dst_id=" + node.node_id + ")"));
- console.assert(cp.node_metadata.plan_node_id == undefined ||
- cp.node_metadata.plan_node_id == node.node_id);
-
- // Add instance events to this node
- if (cp.event_sequences != undefined) {
- for (var en = 0; en < node.events.length; ++en) {
- node.events[en].tslist.push(cp.event_sequences[0].events[en].timestamp);
+ // Use the saved node path to traverse to the same position in this instance
+ for (var pi = 0; pi < node.path.length; ++pi) {
+ cp = cp.child_profiles[node.path[pi]];
+ }
+ console.assert(cp.node_metadata.data_sink_id == undefined ||
+ cp.profile_name.indexOf("(dst_id=" + node.node_id + ")"));
+ console.assert(cp.node_metadata.plan_node_id == undefined ||
+ cp.node_metadata.plan_node_id == node.node_id);
+
+ // Add instance events to this node
+ if (cp.event_sequences != undefined) {
+ for (var en = 0; en < node.events.length; ++en) {
+ node.events[en].tslist.push(cp.event_sequences[0].events[en].timestamp);
+ }
}
}
}
+
+ fragments.push(fragment);
}
+ });
+ profile_available = true;
+ } catch(e) {
+ console.log(e);
+ profile_available = false;
+ }
+}
+
+function renderTiming(ignored_arg) {
+ var plan_order = document.getElementById("plan_order").checked;
+
+ var header_canvas = document.getElementById('header_canvas');
+ var timing_canvas = document.getElementById('timing_canvas');
+ var footer_canvas = document.getElementById('footer_canvas');
+
+ timing_canvas.width = window.innerWidth - 50;
+ header_canvas.width = footer_canvas.width = timing_canvas.clientWidth;
+
+ header_canvas.height = 15; // Height gets resized later
+ timing_canvas.height = 200; // Height gets resized later
+ footer_canvas.height = 15; // Height gets resized later
+
+ var header_ctx = header_canvas.getContext('2d');
+ var timing_ctx = timing_canvas.getContext('2d');
+ var footer_ctx = footer_canvas.getContext('2d');
- fragments.push(fragment);
- }
- });
var frag_name_width = (Math.max(2, (fragments.length - 1).toString().length) + 3) * char_width;
var name_width = max_namelen * char_width + (frag_name_width + 2);
@@ -320,7 +310,8 @@ function renderTiming(ignored_arg) {
var height = 15;
timing_canvas.height = rownum * height;
- var screen_height = Math.min(timing_canvas.height + 10, window.innerHeight - timing_canvas.offsetTop - 30);
+ var screen_height = Math.min(timing_canvas.height + 10,
+ window.innerHeight - timing_canvas.offsetTop - 30);
timing_canvas.parentNode.setAttribute("style",
"height:" + screen_height + "px; overflow-y:auto; border:1px solid #c3c3c3;");
@@ -342,7 +333,7 @@ function renderTiming(ignored_arg) {
header_ctx.fillText(p.label, x + width / 3, text_y, Math.min(text_width, width / 2));
});
- rownum = 0;
+ var rownum_l = 0;
var max_indent = 0;
var pending_children = 0;
var pending_senders = 0;
@@ -359,18 +350,22 @@ function renderTiming(ignored_arg) {
timing_ctx.fillText(frag_name, 1, text_y, frag_name_width);
// Fragment/sender timing row
- DrawBars(timing_ctx, rownum, height, fevents, name_width, px_per_ns);
+ DrawBars(timing_ctx, rownum_l, height, fevents, name_width, px_per_ns);
- for(var i = 0; i < fragment.nodes.length; i++) {
+ for (var i = 0; i < fragment.nodes.length; ++i) {
var node = fragment.nodes[i];
if (node.events != undefined) {
// Plan node timing row
- DrawBars(timing_ctx, rownum, height, node.events, name_width, px_per_ns);
+ DrawBars(timing_ctx, rownum_l, height, node.events, name_width, px_per_ns);
if (node.type == "HASH_JOIN_NODE") {
timing_ctx.fillStyle = "#000000";
- timing_ctx.fillText("X", name_width + minimum(node.events[2].tslist) * px_per_ns, text_y, char_width);
- timing_ctx.fillText("O", name_width + minimum(node.events[2].tslist) * px_per_ns, text_y, char_width);
+ timing_ctx.fillText("X",
+ name_width + Math.min.apply(null, node.events[2].tslist) * px_per_ns,
+ text_y, char_width);
+ timing_ctx.fillText("O",
+ name_width + Math.min.apply(null,node.events[2].tslist) * px_per_ns,
+ text_y, char_width);
}
}
@@ -393,8 +388,7 @@ function renderTiming(ignored_arg) {
if (node.parent_node != undefined) {
var y = height * node.parent_node.rendering.rownum;
if (node.is_sender) {
- var x = name_width + minimum(fevents[3].tslist) * px_per_ns;
-
+ var x = name_width + Math.min.apply(null, fevents[3].tslist) * px_per_ns;
// Dotted horizontal connector to received rows
timing_ctx.beginPath();
timing_ctx.setLineDash([2, 2]);
@@ -404,7 +398,7 @@ function renderTiming(ignored_arg) {
// Dotted rectangle for received rows
timing_ctx.beginPath();
- var x2 = name_width + maximum(fevents[4].tslist) * px_per_ns;
+ var x2 = name_width + Math.max.apply(null, fevents[4].tslist) * px_per_ns;
timing_ctx.strokeRect(x, y + 4, x2 - x, height - 10);
timing_ctx.setLineDash([]);
}
@@ -426,11 +420,11 @@ function renderTiming(ignored_arg) {
timing_ctx.stroke();
}
- node.rendering = { rownum: rownum, label_end: label_x + label_width };
+ node.rendering = { rownum: rownum_l, label_end: label_x + label_width };
if (node.num_children) // Scan (leaf) node
pending_children += (node.num_children - node.is_receiver);
text_y += height;
- rownum++;
+ rownum_l++;
if (node.is_receiver) {
if (plan_order) {
@@ -447,9 +441,11 @@ function renderTiming(ignored_arg) {
}
});
-
- rownum = 0;
- var text_y = (rownum + 1) * height - 4;
+ fragments.forEach(function(fragment) {
+ fragment.printed = false;
+ });
+ rownum_l = 0;
+ var text_y = (rownum_l + 1) * height - 4;
// Time scale below timing diagram
var ntics = 10;
@@ -458,7 +454,7 @@ function renderTiming(ignored_arg) {
var x = name_width;
for (var i = 0; i < ntics; ++i) {
footer_ctx.fillStyle = "#000000";
- var y = rownum * height;
+ var y = rownum_l * height;
footer_ctx.fillRect(x, y, px_per_tic, height);
footer_ctx.fillStyle = "#F0F0F0";
footer_ctx.fillRect(x + 1, y + 1, px_per_tic - 2, height - 2);
@@ -466,27 +462,37 @@ function renderTiming(ignored_arg) {
footer_ctx.fillText((i * sec_per_tic).toFixed(2), x + px_per_tic - 25, text_y, chart_width / ntics);
x += px_per_tic;
}
-
- if (query_finished && intervalId) {
- clearInterval(intervalId);
- intervalId = 0;
- }
}
-
function refresh() {
- req = new XMLHttpRequest();
- req.onload = renderTiming;
+ var req = new XMLHttpRequest();
+ req.onload = function() {
+ if (req.status == 200) {
+ profile = JSON.parse(req.responseText)["profile_json"];
+ collectFromProfile();
+ if (!profile_available) {
+ setTimeout(refresh, 1000);
+ } else {
+ if (profile.child_profiles[0].info_strings.find(({key}) => key === "Query State").value
+ != "FINISHED") {
+ setTimeout(refresh, 1000);
+ }
+ renderTiming();
+ }
+ }
+ }
req.open("GET", make_url("/query_timeline?query_id={{query_id}}&json"), true);
req.send();
}
-// Force one refresh before starting the timer.
-refresh();
+
window.addEventListener('resize', function(event) {
- renderTiming();
+ if (profile_available) {
+ renderTiming();
+ }
}, true);
+window.onload = refresh;
</script>
[impala] 02/02: IMPALA-11950: Planner change for Iceberg metadata querying
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 422006a8090bb7ee4576f4dbb2d6cbd3152ffded
Author: Tamas Mate <tm...@apache.org>
AuthorDate: Mon Feb 27 10:05:51 2023 +0100
IMPALA-11950: Planner change for Iceberg metadata querying
This commit extends the planner with IcebergMetadataScanNode which will
be used to scan Iceberg metadata tables (IMPALA-10947). The scan node is
only implemented on the frontend side in this patch, the backend part
will be developed in IMPALA-11996.
To avoid executing the plan there is a hardcoded condition, it is after
the explain part, so the change remains testable with EXPLAIN queries.
Testing:
- Added planner tests
Change-Id: I3675d7a57ca570bfec306798589b5ef6aa34b5c6
Reviewed-on: http://gerrit.cloudera.org:8080/19547
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../catalog/iceberg/IcebergMetadataTable.java | 20 +++-
.../apache/impala/planner/DistributedPlanner.java | 10 ++
.../impala/planner/IcebergMetadataScanNode.java | 78 ++++++++++++++
.../java/org/apache/impala/planner/ScanNode.java | 6 +-
.../apache/impala/planner/SingleNodePlanner.java | 13 ++-
.../java/org/apache/impala/service/Frontend.java | 14 ++-
.../org/apache/impala/planner/PlannerTest.java | 9 ++
.../PlannerTest/iceberg-metadata-table-scan.test | 114 +++++++++++++++++++++
8 files changed, 251 insertions(+), 13 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergMetadataTable.java b/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergMetadataTable.java
index 42756c62c..b7d0dcbe0 100644
--- a/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergMetadataTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/iceberg/IcebergMetadataTable.java
@@ -25,6 +25,7 @@ import org.apache.iceberg.MetadataTableType;
import org.apache.iceberg.MetadataTableUtils;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
+import org.apache.impala.analysis.TableName;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.FeCatalogUtils;
import org.apache.impala.catalog.FeIcebergTable;
@@ -45,12 +46,14 @@ import com.google.common.base.Preconditions;
*/
public class IcebergMetadataTable extends VirtualTable {
private FeIcebergTable baseTable_;
+ private String metadataTableName_;
public IcebergMetadataTable(FeTable baseTable, String metadataTableTypeStr)
throws ImpalaRuntimeException {
super(null, baseTable.getDb(), baseTable.getName(), baseTable.getOwnerUser());
Preconditions.checkArgument(baseTable instanceof FeIcebergTable);
baseTable_ = (FeIcebergTable) baseTable;
+ metadataTableName_ = metadataTableTypeStr;
MetadataTableType type = MetadataTableType.from(metadataTableTypeStr.toUpperCase());
Preconditions.checkNotNull(type);
Table metadataTable = MetadataTableUtils.createMetadataTableInstance(
@@ -71,6 +74,16 @@ public class IcebergMetadataTable extends VirtualTable {
return baseTable_;
}
+ @Override
+ public String getFullName() {
+ return super.getFullName() + "." + metadataTableName_;
+ }
+
+ @Override
+ public TableName getTableName() {
+ return new TableName(db_.getName(), name_, metadataTableName_);
+ }
+
@Override
public TTableStats getTTableStats() {
long totalBytes = 0;
@@ -79,9 +92,14 @@ public class IcebergMetadataTable extends VirtualTable {
return ret;
}
+ @Override
+ public org.apache.hadoop.hive.metastore.api.Table getMetaStoreTable() {
+ return baseTable_.getMetaStoreTable();
+ }
+
/**
* Return same descriptor as the base table, but with a schema that corresponds to
- * the metadtata table schema.
+ * the metadata table schema.
*/
@Override
public TTableDescriptor toThriftDescriptor(int tableId,
diff --git a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
index 67a376a7e..56e54a7b2 100644
--- a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
@@ -147,6 +147,9 @@ public class DistributedPlanner {
ctx_.getNextFragmentId(), root, DataPartition.UNPARTITIONED);
} else if (root instanceof CardinalityCheckNode) {
result = createCardinalityCheckNodeFragment((CardinalityCheckNode) root, childFragments);
+ } else if (root instanceof IcebergMetadataScanNode) {
+ result = createIcebergMetadataScanFragment(root);
+ fragments.add(result);
} else {
throw new InternalException("Cannot create plan fragment for this node type: "
+ root.getExplainString(ctx_.getQueryOptions()));
@@ -325,6 +328,13 @@ public class DistributedPlanner {
return new PlanFragment(ctx_.getNextFragmentId(), node, DataPartition.RANDOM);
}
+ /**
+ * Create an Iceberg Metadata scan fragment.
+ */
+ private PlanFragment createIcebergMetadataScanFragment(PlanNode node) {
+ return new PlanFragment(ctx_.getNextFragmentId(), node, DataPartition.UNPARTITIONED);
+ }
+
/**
* Adds the SubplanNode as the new plan root to the child fragment and returns
* the child fragment.
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergMetadataScanNode.java b/fe/src/main/java/org/apache/impala/planner/IcebergMetadataScanNode.java
new file mode 100644
index 000000000..4a776786e
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergMetadataScanNode.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.planner;
+
+import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.TupleDescriptor;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.thrift.TExplainLevel;
+import org.apache.impala.thrift.TPlanNode;
+import org.apache.impala.thrift.TQueryOptions;
+import org.apache.impala.thrift.TScanRangeSpec;
+
+import com.google.common.base.Preconditions;
+
+public class IcebergMetadataScanNode extends ScanNode {
+
+ protected IcebergMetadataScanNode(PlanNodeId id, TupleDescriptor desc) {
+ super(id, desc, "SCAN ICEBERG METADATA");
+ }
+
+ @Override
+ public void init(Analyzer analyzer) throws ImpalaException {
+ super.init(analyzer);
+ scanRangeSpecs_ = new TScanRangeSpec();
+ computeMemLayout(analyzer);
+ computeStats(analyzer);
+ }
+
+ @Override
+ protected String getDisplayLabelDetail() {
+ Preconditions.checkNotNull(desc_.getPath());
+ if (desc_.hasExplicitAlias()) {
+ return desc_.getPath().toString() + " " + desc_.getAlias();
+ } else {
+ return desc_.getPath().toString();
+ }
+ }
+
+ @Override
+ protected String getNodeExplainString(String rootPrefix, String detailPrefix,
+ TExplainLevel detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(String.format("%s%s [%s]\n", rootPrefix, getDisplayLabel(),
+ getDisplayLabelDetail()));
+ return output.toString();
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ // Implement for fragment execution
+ }
+
+ @Override
+ public void computeNodeResourceProfile(TQueryOptions queryOptions) {
+ nodeResourceProfile_ = ResourceProfile.noReservation(0);
+ }
+
+ @Override
+ public void computeProcessingCost(TQueryOptions queryOptions) {
+ processingCost_ = computeDefaultProcessingCost();
+ }
+
+}
diff --git a/fe/src/main/java/org/apache/impala/planner/ScanNode.java b/fe/src/main/java/org/apache/impala/planner/ScanNode.java
index 59ebb179c..d9b3ac49b 100644
--- a/fe/src/main/java/org/apache/impala/planner/ScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/ScanNode.java
@@ -310,10 +310,6 @@ abstract public class ScanNode extends PlanNode {
@Override
protected String getDisplayLabelDetail() {
- FeTable table = desc_.getTable();
- List<String> path = new ArrayList<>();
- path.add(table.getDb().getName());
- path.add(table.getName());
Preconditions.checkNotNull(desc_.getPath());
if (desc_.hasExplicitAlias()) {
return desc_.getPath().toString() + " " + desc_.getAlias();
@@ -391,7 +387,7 @@ abstract public class ScanNode extends PlanNode {
/**
* Estimate per-row cost as 1 per 1KB row size.
* <p>
- * This reflect deserialization/copy cost per row.
+ * This reflects deserialization/copy cost per row.
*/
private float rowMaterializationCost() { return getAvgRowSize() / 1024; }
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 388d08094..71c7af2e5 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -863,7 +863,6 @@ public class SingleNodePlanner {
// If the collection is within a (possibly nested) struct, add the tuple in
// which the top level struct is located.
SlotDescriptor desc = collectionExpr.getDesc();
- List<TupleDescriptor> enclosingTupleDescs = desc.getEnclosingTupleDescs();
TupleDescriptor topTuple = desc.getTopEnclosingTupleDesc();
requiredTids.add(topTuple.getId());
} else {
@@ -1900,6 +1899,14 @@ public class SingleNodePlanner {
}
}
+ private PlanNode createIcebergMetadataScanNode(TableRef tblRef, Analyzer analyzer)
+ throws ImpalaException {
+ IcebergMetadataScanNode icebergMetadataScanNode =
+ new IcebergMetadataScanNode(ctx_.getNextNodeId(), tblRef.getDesc());
+ icebergMetadataScanNode.init(analyzer);
+ return icebergMetadataScanNode;
+ }
+
/**
* Returns all applicable conjuncts for join between two plan trees 'materializing' the
* given left-hand and right-hand side table ref ids. The conjuncts either come from
@@ -2213,9 +2220,7 @@ public class SingleNodePlanner {
result = new SingularRowSrcNode(ctx_.getNextNodeId(), ctx_.getSubplan());
result.init(analyzer);
} else if (tblRef instanceof IcebergMetadataTableRef) {
- throw new NotImplementedException(String.format("'%s' refers to a metadata table "
- + "which is currently not supported.", String.join(".",
- tblRef.getPath())));
+ result = createIcebergMetadataScanNode(tblRef, analyzer);
} else {
throw new NotImplementedException(
"Planning not implemented for table ref class: " + tblRef.getClass());
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index d51e34897..8829beab9 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -124,6 +124,7 @@ import org.apache.impala.catalog.ImpaladTableUsageTracker;
import org.apache.impala.catalog.MaterializedViewHdfsTable;
import org.apache.impala.catalog.MetaStoreClientPool;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
+import org.apache.impala.catalog.iceberg.IcebergMetadataTable;
import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.local.InconsistentMetadataFetchException;
@@ -290,9 +291,8 @@ public class Frontend {
protected long estimated_memory_per_host_ = -1;
// The processing cores required to execute the query.
- // Certain queries such as EXPLAIN that do not populate
- // TExecRequest.query_exec_request. Therefore, cores requirement will be set here
- // through setCoresRequired().
+ // Certain queries such as EXPLAIN do not populate TExecRequest.query_exec_request.
+ // Therefore, cores requirement will be set here through setCoresRequired().
protected int cores_required_ = -1;
// The initial length of content in explain buffer to help return the buffer
@@ -2470,6 +2470,14 @@ public class Frontend {
return result;
}
+ // Blocking query execution for queries that contain IcebergMetadataTables
+ for (FeTable table : stmtTableCache.tables.values()) {
+ if (table instanceof IcebergMetadataTable) {
+ throw new NotImplementedException(String.format("'%s' refers to a metadata "
+ + "table which is currently not supported.", table.getFullName()));
+ }
+ }
+
result.setQuery_exec_request(queryExecRequest);
if (analysisResult.isQueryStmt()) {
result.stmt_type = TStmtType.QUERY;
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index 6b4eba927..a3b91eaf5 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -1282,6 +1282,15 @@ public class PlannerTest extends PlannerTestBase {
ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
}
+ /**
+ * Check that Iceberg metadata table scan plans are as expected.
+ */
+ @Test
+ public void testIcebergMetadataTableScans() {
+ runPlannerTestFile("iceberg-metadata-table-scan", "functional_parquet",
+ ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
+ }
+
/**
* Test limit pushdown into analytic sort in isolation.
*/
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-metadata-table-scan.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-metadata-table-scan.test
new file mode 100644
index 000000000..f976cb75e
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-metadata-table-scan.test
@@ -0,0 +1,114 @@
+explain SELECT * FROM functional_parquet.iceberg_alltypes_part_orc.history
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history]
+ row-size=33B cardinality=unavailable
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history]
+ row-size=33B cardinality=unavailable
+====
+explain select *
+from functional_parquet.iceberg_alltypes_part_orc.history q
+ join functional_parquet.iceberg_alltypes_part_orc.history z
+ on z.snapshot_id = q.snapshot_id
+---- PLAN
+PLAN-ROOT SINK
+|
+02:NESTED LOOP JOIN [CROSS JOIN]
+| row-size=66B cardinality=unavailable
+|
+|--01:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history z]
+| row-size=33B cardinality=unavailable
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history q]
+ row-size=33B cardinality=unavailable
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+04:EXCHANGE [UNPARTITIONED]
+|
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| row-size=66B cardinality=unavailable
+|
+|--03:EXCHANGE [BROADCAST]
+| |
+| 01:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history z]
+| row-size=33B cardinality=unavailable
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history q]
+ row-size=33B cardinality=unavailable
+====
+explain select q.snapshot_id, z.made_current_at as test1, z.made_current_at as test2
+from functional_parquet.iceberg_alltypes_part_orc.history q
+ join functional_parquet.iceberg_alltypes_part_orc.history z
+ on z.snapshot_id = q.snapshot_id
+---- PLAN
+PLAN-ROOT SINK
+|
+02:NESTED LOOP JOIN [CROSS JOIN]
+| row-size=24B cardinality=unavailable
+|
+|--01:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history z]
+| row-size=16B cardinality=unavailable
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history q]
+ row-size=8B cardinality=unavailable
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+04:EXCHANGE [UNPARTITIONED]
+|
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| row-size=24B cardinality=unavailable
+|
+|--03:EXCHANGE [BROADCAST]
+| |
+| 01:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history z]
+| row-size=16B cardinality=unavailable
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.history q]
+ row-size=8B cardinality=unavailable
+====
+explain select * from functional_parquet.iceberg_alltypes_part_orc.manifests a, a.partition_summaries
+---- PLAN
+PLAN-ROOT SINK
+|
+01:SUBPLAN
+| row-size=98B cardinality=unavailable
+|
+|--04:NESTED LOOP JOIN [CROSS JOIN]
+| | row-size=98B cardinality=10
+| |
+| |--02:SINGULAR ROW SRC
+| | row-size=72B cardinality=1
+| |
+| 03:UNNEST [a.partition_summaries]
+| row-size=0B cardinality=10
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.manifests a]
+ row-size=72B cardinality=unavailable
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+05:EXCHANGE [UNPARTITIONED]
+|
+01:SUBPLAN
+| row-size=98B cardinality=unavailable
+|
+|--04:NESTED LOOP JOIN [CROSS JOIN]
+| | row-size=98B cardinality=10
+| |
+| |--02:SINGULAR ROW SRC
+| | row-size=72B cardinality=1
+| |
+| 03:UNNEST [a.partition_summaries]
+| row-size=0B cardinality=10
+|
+00:SCAN ICEBERG METADATA [functional_parquet.iceberg_alltypes_part_orc.manifests a]
+ row-size=72B cardinality=unavailable
+====
\ No newline at end of file