You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ry...@apache.org on 2021/04/09 16:45:46 UTC
[airflow] branch master updated: Migrate graph js (#15307)
This is an automated email from the ASF dual-hosted git repository.
ryanahamilton pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new bcc6c93 Migrate graph js (#15307)
bcc6c93 is described below
commit bcc6c9331466c32cf9c0ec87ac787e574dbd7208
Author: Brent Bovenzi <br...@gmail.com>
AuthorDate: Fri Apr 9 11:45:27 2021 -0500
Migrate graph js (#15307)
* move graph js to its own file
* remove some eslint-disable
* keep node update
---
airflow/www/static/js/graph.js | 704 ++++++++++++++++++++++++++++++
airflow/www/templates/airflow/graph.html | 713 +------------------------------
airflow/www/webpack.config.js | 2 +-
3 files changed, 720 insertions(+), 699 deletions(-)
diff --git a/airflow/www/static/js/graph.js b/airflow/www/static/js/graph.js
new file mode 100644
index 0000000..eda6c19
--- /dev/null
+++ b/airflow/www/static/js/graph.js
@@ -0,0 +1,704 @@
+/* eslint-disable no-underscore-dangle */
+/* eslint-disable no-use-before-define */
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ global d3, document, call_modal, nodes, taskInstances, tasks, edges, dagreD3, localStorage, $
+*/
+
+import getMetaValue from './meta_value';
+import { escapeHtml } from './main';
+import tiTooltip, { taskNoInstanceTooltip } from './task_instances';
+
+const dagId = getMetaValue('dag_id');
+const executionDate = getMetaValue('execution_date');
+const arrange = getMetaValue('arrange');
+const taskInstancesUrl = getMetaValue('task_instances_url');
+
+// Build a map mapping node id to tooltip for all the TaskGroups.
+function getTaskGroupTips(node) {
+ const tips = new Map();
+ if (node.children !== undefined) {
+ tips.set(node.id, node.tooltip);
+ for (const child of node.children.values()) {
+ for (const [key, val] of getTaskGroupTips(child)) tips.set(key, val);
+ }
+ }
+ return tips;
+}
+
+const taskGroupTips = getTaskGroupTips(nodes);
+// This maps the actual taskId to the current graph node id that contains the task
+// (because tasks may be grouped into a group node)
+const mapTaskToNode = new Map();
+
+// Below variables are being used in dag.js
+
+const getTaskInstanceURL = `${taskInstancesUrl}?dag_id=${encodeURIComponent(dagId)}&execution_date=${
+ encodeURIComponent(executionDate)}`;
+
+const duration = 500;
+const stateFocusMap = {
+ success: false,
+ running: false,
+ failed: false,
+ skipped: false,
+ upstream_failed: false,
+ up_for_reschedule: false,
+ up_for_retry: false,
+ queued: false,
+ no_status: false,
+};
+const taskTip = d3.tip()
+ .attr('class', 'tooltip d3-tip')
+ .html((toolTipHtml) => toolTipHtml);
+
+// Preparation of DagreD3 data structures
+// "compound" is set to true to make use of clusters to display TaskGroup.
+const g = new dagreD3.graphlib.Graph({ compound: true }).setGraph({
+ nodesep: 30,
+ ranksep: 15,
+ rankdir: arrange,
+})
+ .setDefaultEdgeLabel(() => ({ lineInterpolate: 'basis' }));
+
+const render = dagreD3.render();
+const svg = d3.select('#graph-svg');
+let innerSvg = d3.select('#graph-svg g');
+
+// Remove the node with this nodeId from g.
+function removeNode(nodeId) {
+ if (g.hasNode(nodeId)) {
+ const node = g.node(nodeId);
+ if (node.children !== undefined) {
+ // If the child is an expanded group node, remove children too.
+ node.children.forEach((child) => {
+ removeNode(child.id);
+ });
+ }
+ }
+ g.removeNode(nodeId);
+}
+
+// Collapse the children of the given group node.
+function collapseGroup(nodeId, node) {
+ // Remove children nodes
+ node.children.forEach((child) => {
+ removeNode(child.id);
+ });
+ // Map task that are under this node to this node's id
+ for (const childId of getChildrenIds(node)) mapTaskToNode.set(childId, nodeId);
+
+ node = g.node(nodeId);
+
+ // Set children edges onto the group edge
+ edges.forEach((edge) => {
+ const sourceId = mapTaskToNode.get(edge.source_id);
+ const targetId = mapTaskToNode.get(edge.target_id);
+ if (sourceId !== targetId && !g.hasEdge(sourceId, targetId)) {
+ g.setEdge(sourceId, targetId, {
+ curve: d3.curveBasis,
+ arrowheadClass: 'arrowhead',
+ });
+ }
+ });
+
+ draw();
+ focusGroup(nodeId);
+
+ removeExpandedGroup(nodeId, node);
+}
+
+// Update the page to show the latest DAG.
+function draw() {
+ innerSvg.remove();
+ innerSvg = svg.append('g');
+ // Run the renderer. This is what draws the final graph.
+ innerSvg.call(render, g);
+ innerSvg.call(taskTip);
+
+ // When an expanded group is clicked, collapse it.
+ d3.selectAll('g.cluster').on('click', (nodeId) => {
+ if (d3.event.defaultPrevented) return;
+ const node = g.node(nodeId);
+ collapseGroup(nodeId, node);
+ });
+ // When a node is clicked, action depends on the node type.
+ d3.selectAll('g.node').on('click', (nodeId) => {
+ const node = g.node(nodeId);
+ if (node.children !== undefined && Object.keys(node.children).length > 0) {
+ // A group node
+ if (d3.event.defaultPrevented) return;
+ expandGroup(nodeId, node);
+ } else if (nodeId in tasks) {
+ // A task node
+ const task = tasks[nodeId];
+ let tryNumber;
+ if (nodeId in taskInstances) tryNumber = taskInstances[nodeId].tryNumber;
+ else tryNumber = 0;
+
+ if (task.task_type === 'SubDagOperator') call_modal(nodeId, executionDate, task.extra_links, tryNumber, true);
+ else call_modal(nodeId, executionDate, task.extra_links, tryNumber, undefined);
+ } else {
+ // join node between TaskGroup. Ignore.
+ }
+ });
+
+ d3.selectAll('g.node').on('mouseover', function mousover(d) {
+ d3.select(this).selectAll('rect').attr('data-highlight', 'highlight');
+ highlightNodes(g.predecessors(d));
+ highlightNodes(g.successors(d));
+ const adjacentNodeNames = [d, ...g.predecessors(d), ...g.successors(d)];
+
+ d3.selectAll('g.nodes g.node')
+ .filter((x) => !adjacentNodeNames.includes(x))
+ .attr('data-highlight', 'fade');
+
+ d3.selectAll('g.edgePath')[0].forEach((x) => {
+ const val = g.nodeEdges(d).includes(x.__data__) ? 'highlight' : 'fade';
+ d3.select(x).attr('data-highlight', val);
+ });
+ d3.selectAll('g.edgeLabel')[0].forEach((x) => {
+ if (!g.nodeEdges(d).includes(x.__data__)) {
+ d3.select(x).attr('data-highlight', 'fade');
+ }
+ });
+ });
+
+ d3.selectAll('g.node').on('mouseout', function mouseout(d) {
+ d3.select(this).selectAll('rect, circle').attr('data-highlight', null);
+ unHighlightNodes(g.predecessors(d));
+ unHighlightNodes(g.successors(d));
+ d3.selectAll('g.node, g.edgePath, g.edgeLabel')
+ .attr('data-highlight', null);
+ localStorage.removeItem(focusedGroupKey(dagId));
+ });
+ updateNodesStates(taskInstances);
+ setUpZoomSupport();
+}
+
+let zoom = null;
+
+function setUpZoomSupport() {
+ // Set up zoom support for Graph
+ zoom = d3.behavior.zoom().on('zoom', () => {
+ innerSvg.attr('transform', `translate(${d3.event.translate})scale(${d3.event.scale})`);
+ });
+ svg.call(zoom);
+
+ // Centering the DAG on load
+ // Get Dagre Graph dimensions
+ const graphWidth = g.graph().width;
+ const graphHeight = g.graph().height;
+ // Get SVG dimensions
+ const padding = 20;
+ const svgBb = svg.node().getBoundingClientRect();
+ const width = svgBb.width - padding * 2;
+ const height = svgBb.height - padding; // we are not centering the dag vertically
+
+ // Calculate applicable scale for zoom
+ const zoomScale = Math.min(
+ Math.min(width / graphWidth, height / graphHeight),
+ 1.5, // cap zoom level to 1.5 so nodes are not too large
+ );
+
+ zoom.translate([(width / 2) - ((graphWidth * zoomScale) / 2) + padding, padding]);
+ zoom.scale(zoomScale);
+ zoom.event(innerSvg);
+}
+
+function highlightNodes(nodes) {
+ nodes.forEach((nodeid) => {
+ const myNode = g.node(nodeid).elem;
+ d3.select(myNode)
+ .selectAll('rect, circle')
+ .attr('data-highlight', 'highlight');
+ });
+}
+
+function unHighlightNodes(nodes) {
+ nodes.forEach((nodeid) => {
+ const myNode = g.node(nodeid).elem;
+ d3.select(myNode)
+ .selectAll('rect, circle')
+ .attr('data-highlight', null);
+ });
+}
+
+d3.selectAll('.js-state-legend-item')
+ .on('mouseover', function mouseover() {
+ if (!stateIsSet()) {
+ const state = $(this).data('state');
+ focusState(state);
+ }
+ })
+ .on('mouseout', () => {
+ if (!stateIsSet()) {
+ clearFocus();
+ }
+ });
+
+d3.selectAll('.js-state-legend-item').on('click', function click() {
+ const state = $(this).data('state');
+
+ clearFocus();
+ if (!stateFocusMap[state]) {
+ const color = d3.select(this).style('border-color');
+ focusState(state, this, color);
+ setFocusMap(state);
+ } else {
+ setFocusMap();
+ d3.selectAll('.js-state-legend-item')
+ .style('background-color', null);
+ }
+});
+
+// Returns true if a node's id or its children's id matches searchText
+function nodeMatches(nodeId, searchText) {
+ if (nodeId.indexOf(searchText) > -1) return true;
+
+ // The node's own id does not match, it may have children that match
+ const node = g.node(nodeId);
+ if (node.children !== undefined) {
+ const children = getChildrenIds(node);
+ for (const child of children) {
+ if (child.indexOf(searchText) > -1) return true;
+ }
+ }
+ return false;
+}
+
+d3.select('#searchbox').on('keyup', () => {
+ const s = document.getElementById('searchbox').value;
+
+ if (s === '') return;
+
+ let match = null;
+
+ if (stateIsSet()) {
+ clearFocus();
+ setFocusMap();
+ }
+
+ d3.selectAll('g.nodes g.node').forEach(function highlight(d) {
+ if (s === '') {
+ d3.selectAll('g.edgePaths, g.edgeLabel').attr('data-highlight', null);
+ d3.select(this).attr('data-highlight', null);
+ } else {
+ d3.selectAll('g.edgePaths, g.edgeLabel').attr('data-highlight', 'fade');
+ if (nodeMatches(d, s)) {
+ if (!match) match = this;
+ d3.select(this).attr('data-highlight', null);
+ } else {
+ d3.select(this).attr('data-highlight', 'fade');
+ }
+ }
+ });
+
+ // This moves the matched node to the center of the graph area
+ if (match) {
+ const transform = d3.transform(d3.select(match).attr('transform'));
+
+ const svgBb = svg.node().getBoundingClientRect();
+ transform.translate = [
+ svgBb.width / 2 - transform.translate[0],
+ svgBb.height / 2 - transform.translate[1],
+ ];
+ transform.scale = [1, 1];
+
+ if (zoom != null) {
+ zoom.translate(transform.translate);
+ zoom.scale(1);
+ zoom.event(innerSvg);
+ }
+ }
+});
+
+function clearFocus() {
+ d3.selectAll('g.node, g.edgePaths, g.edgeLabel')
+ .attr('data-highlight', null);
+ localStorage.removeItem(focusedGroupKey(dagId));
+}
+
+function focusState(state, node, color) {
+ d3.selectAll('g.node, g.edgePaths, g.edgeLabel')
+ .attr('data-highlight', 'fade');
+ d3.selectAll(`g.node.${state}`)
+ .attr('data-highlight', null);
+ d3.selectAll(`g.node.${state} rect`)
+ .attr('data-highlight', null);
+ d3.select(node)
+ .style('background-color', color);
+}
+
+function setFocusMap(state) {
+ for (const key in stateFocusMap) {
+ if ({}.hasOwnProperty.call(stateFocusMap, key)) {
+ stateFocusMap[key] = false;
+ }
+ }
+ if (state != null) {
+ stateFocusMap[state] = true;
+ }
+}
+
+function stateIsSet() {
+ for (const key in stateFocusMap) {
+ if (stateFocusMap[key]) {
+ return true;
+ }
+ }
+ return false;
+}
+
+function handleRefresh() {
+ $('#loading-dots').css('display', 'inline-block');
+ $.get(getTaskInstanceURL)
+ .done(
+ (tis) => {
+ // eslint-disable-next-line no-global-assign
+ taskInstances = JSON.parse(tis);
+ updateNodesStates(taskInstances);
+ setTimeout(() => { $('#loading-dots').hide(); }, 500);
+ $('#error').hide();
+ },
+ ).fail((_, textStatus, err) => {
+ $('#error_msg').text(`${textStatus}: ${err}`);
+ $('#error').show();
+ setTimeout(() => { $('#loading-dots').hide(); }, 500);
+ $('#chart_section').hide(1000);
+ $('#datatable_section').hide(1000);
+ });
+}
+
+let refreshInterval;
+
+function startOrStopRefresh() {
+ if ($('#auto_refresh').is(':checked')) {
+ refreshInterval = setInterval(() => {
+ handleRefresh();
+ }, 3000); // run refresh every 3 seconds
+ } else {
+ clearInterval(refreshInterval);
+ }
+}
+
+$('#auto_refresh').change(() => {
+ if ($('#auto_refresh').is(':checked')) {
+ // Run an initial refesh before starting interval if manually turned on
+ handleRefresh();
+ localStorage.removeItem('disableAutoRefresh');
+ } else {
+ localStorage.setItem('disableAutoRefresh', 'true');
+ }
+ startOrStopRefresh();
+});
+
+function initRefresh() {
+ if (localStorage.getItem('disableAutoRefresh')) {
+ $('#auto_refresh').removeAttr('checked');
+ }
+ startOrStopRefresh();
+ d3.select('#refresh_button').on('click', () => handleRefresh());
+}
+
+// Generate tooltip for a group node
+function groupTooltip(nodeId, tis) {
+ const numMap = new Map([['success', 0],
+ ['failed', 0],
+ ['upstream_failed', 0],
+ ['up_for_retry', 0],
+ ['running', 0],
+ ['no_status', 0]]);
+ for (const child of getChildrenIds(g.node(nodeId))) {
+ if (child in tis) {
+ const ti = tis[child];
+ const stateKey = ti.state == null ? 'no_status' : ti.state;
+ if (numMap.has(stateKey)) numMap.set(stateKey, numMap.get(stateKey) + 1);
+ }
+ }
+
+ const tip = taskGroupTips.get(nodeId);
+ let tt = `${escapeHtml(tip)}<br><br>`;
+ for (const [key, val] of numMap.entries()) tt += `<strong>${escapeHtml(key)}:</strong> ${val} <br>`;
+
+ return tt;
+}
+
+// Assigning css classes based on state to nodes
+// Initiating the tooltips
+function updateNodesStates(tis) {
+ for (const nodeId of g.nodes()) {
+ const { elem } = g.node(nodeId);
+ elem.setAttribute('class', `node enter ${getNodeState(nodeId, tis)}`);
+ elem.setAttribute('data-toggle', 'tooltip');
+
+ const taskId = nodeId;
+ elem.onmouseover = (evt) => {
+ if (taskId in tis) {
+ const tt = tiTooltip(tis[taskId]);
+ taskTip.show(tt, evt.target); // taskTip is defined in graph.html
+ } else if (taskGroupTips.has(taskId)) {
+ const tt = groupTooltip(taskId, tis);
+ taskTip.show(tt, evt.target);
+ } else if (taskId in tasks) {
+ const tt = taskNoInstanceTooltip(taskId, tasks[taskId]);
+ taskTip.show(tt, evt.target);
+ }
+ };
+ elem.onmouseout = taskTip.hide;
+ elem.onclick = taskTip.hide;
+ }
+}
+
+// Returns list of children id of the given task group
+function getChildrenIds(group) {
+ const children = [];
+ for (const [key, val] of Object.entries(group.children)) {
+ if (val.children === undefined) {
+ // node
+ children.push(val.id);
+ } else {
+ // group
+ const subGroupChildren = getChildrenIds(val);
+ for (const id of subGroupChildren) {
+ children.push(id);
+ }
+ }
+ }
+ return children;
+}
+
+// Return list of all task group ids in the given task group including the given group itself.
+function getAllGroupIds(group) {
+ const children = [group.id];
+
+ for (const [key, val] of Object.entries(group.children)) {
+ if (val.children !== undefined) {
+ // group
+ const subGroupChildren = getAllGroupIds(val);
+ for (const id of subGroupChildren) {
+ children.push(id);
+ }
+ }
+ }
+ return children;
+}
+
+// Return the state for the node based on the state of its taskinstance or that of its
+// children if it's a group node
+function getNodeState(nodeId, tis) {
+ const node = g.node(nodeId);
+
+ if (node.children === undefined) {
+ if (nodeId in tis) {
+ return tis[nodeId].state || 'no_status';
+ }
+ return 'no_status';
+ }
+ const children = getChildrenIds(node);
+
+ const childrenStates = new Set();
+ children.forEach((taskId) => {
+ if (taskId in tis) {
+ const { state } = tis[taskId];
+ childrenStates.add(state == null ? 'no_status' : state);
+ }
+ });
+
+ // In this order, if any of these states appeared in childrenStates, return it as
+ // the group state.
+ const priority = ['failed', 'upstream_failed', 'up_for_retry', 'up_for_reschedule',
+ 'queued', 'scheduled', 'sensing', 'running', 'shutdown', 'removed',
+ 'no_status', 'success', 'skipped'];
+
+ for (const state of priority) {
+ if (childrenStates.has(state)) return state;
+ }
+ return 'no_status';
+}
+
+// Returns the key used to store expanded task group ids in localStorage
+function expandedGroupsKey() {
+ return `expandedGroups_${dagId}`;
+}
+
+// Returns the key used to store the focused task group id in localStorage
+function focusedGroupKey() {
+ return `focused_group_${dagId}`;
+}
+
+// Focus the graph on the expanded/collapsed node
+function focusGroup(nodeId) {
+ if (nodeId != null && zoom != null) {
+ const { x } = g.node(nodeId);
+ const { y } = g.node(nodeId);
+ // This is the total canvas size.
+ const { width, height } = svg.node().getBoundingClientRect();
+
+ // This is the size of the node or the cluster (i.e. group)
+ let rect = d3.selectAll('g.node').filter((n) => n === nodeId).select('rect');
+ if (rect.empty()) rect = d3.selectAll('g.cluster').filter((n) => n === nodeId).select('rect');
+
+ // Is there a better way to get nodeWidth and nodeHeight ?
+ const [nodeWidth, nodeHeight] = [
+ rect[0][0].attributes.width.value, rect[0][0].attributes.height.value,
+ ];
+
+ // Calculate zoom scale to fill most of the canvas with the node/cluster in focus.
+ const scale = Math.min(
+ Math.min(width / nodeWidth, height / nodeHeight),
+ 1.5, // cap zoom level to 1.5 so nodes are not too large
+ ) * 0.9;
+
+ const [deltaX, deltaY] = [width / 2 - x * scale, height / 2 - y * scale];
+ zoom.translate([deltaX, deltaY]);
+ zoom.scale(scale);
+ zoom.event(innerSvg.transition().duration(duration));
+
+ const children = new Set(g.children(nodeId));
+ // Set data attr to highlight the focused group (via CSS).
+ d3.selectAll('g.nodes g.node').forEach(function cssHighlight(d) {
+ if (d === nodeId || children.has(d)) {
+ d3.select(this)
+ .attr('data-highlight', null);
+ } else {
+ d3.select(this)
+ .attr('data-highlight', 'fade');
+ }
+ });
+
+ localStorage.setItem(focusedGroupKey(dagId), nodeId);
+ }
+}
+
+// Expands a group node
+function expandGroup(nodeId, node, focus = true) {
+ node.children.forEach((val) => {
+ // Set children nodes
+ g.setNode(val.id, val.value);
+ mapTaskToNode.set(val.id, val.id);
+ g.node(val.id).id = val.id;
+ if (val.children !== undefined) {
+ // Set children attribute so that the group can be expanded later when needed.
+ const groupNode = g.node(val.id);
+ groupNode.children = val.children;
+ // Map task that are under this node to this node's id
+ for (const childId of getChildrenIds(val)) mapTaskToNode.set(childId, val.id);
+ }
+ // Only call setParent if node is not the root node.
+ if (nodeId != null) g.setParent(val.id, nodeId);
+ });
+
+ // Add edges
+ edges.forEach((edge) => {
+ const sourceId = mapTaskToNode.get(edge.source_id);
+ const targetId = mapTaskToNode.get(edge.target_id);
+ if (sourceId !== targetId && !g.hasEdge(sourceId, targetId)) {
+ g.setEdge(sourceId, targetId, {
+ curve: d3.curveBasis,
+ arrowheadClass: 'arrowhead',
+ label: edge.label,
+ });
+ }
+ });
+
+ g.edges().forEach((edge) => {
+ // Remove edges that were associated with the expanded group node..
+ if (nodeId === edge.v || nodeId === edge.w) {
+ g.removeEdge(edge.v, edge.w);
+ }
+ });
+
+ draw();
+
+ if (focus) {
+ focusGroup(nodeId);
+ }
+
+ saveExpandedGroup(nodeId);
+}
+
+function getSavedGroups() {
+ let expandedGroups;
+ try {
+ expandedGroups = new Set(JSON.parse(localStorage.getItem(expandedGroupsKey(dagId))));
+ } catch {
+ expandedGroups = new Set();
+ }
+
+ return expandedGroups;
+}
+
+// Clean up invalid group_ids from saved_group_ids (e.g. due to DAG changes)
+function pruneInvalidSavedGroupIds() {
+ // All the groupIds in the whole DAG
+ const allGroupIds = new Set(getAllGroupIds(nodes));
+ let expandedGroups = getSavedGroups(dagId);
+ expandedGroups = Array.from(expandedGroups).filter((groupId) => allGroupIds.has(groupId));
+ localStorage.setItem(expandedGroupsKey(dagId), JSON.stringify(expandedGroups));
+}
+
+// Remember the expanded groups in local storage so that it can be used
+// to restore the expanded state of task groups.
+function saveExpandedGroup(nodeId) {
+ // expandedGroups is a Set
+ const expandedGroups = getSavedGroups(dagId);
+ expandedGroups.add(nodeId);
+ localStorage.setItem(expandedGroupsKey(dagId), JSON.stringify(Array.from(expandedGroups)));
+}
+
+// Remove the nodeId from the expanded state
+function removeExpandedGroup(nodeId, node) {
+ const expandedGroups = getSavedGroups(dagId);
+ const childGroupIds = getAllGroupIds(node);
+ childGroupIds.forEach((childId) => expandedGroups.delete(childId));
+ localStorage.setItem(expandedGroupsKey(dagId), JSON.stringify(Array.from(expandedGroups)));
+}
+
+// Restore previously expanded task groups
+function expandSavedGroups(expandedGroups, node) {
+ if (node.children === undefined) return;
+
+ node.children.forEach((childNode) => {
+ if (expandedGroups.has(childNode.id)) {
+ expandGroup(childNode.id, g.node(childNode.id), false);
+
+ expandSavedGroups(expandedGroups, childNode);
+ }
+ });
+}
+
+pruneInvalidSavedGroupIds();
+const focusNodeId = localStorage.getItem(focusedGroupKey(dagId));
+const expandedGroups = getSavedGroups(dagId);
+
+// Always expand the root node
+expandGroup(null, nodes);
+
+// Expand the node that were previously expanded
+expandSavedGroups(expandedGroups, nodes);
+
+// Restore focus (if available)
+if (g.hasNode(focusNodeId)) {
+ focusGroup(focusNodeId);
+}
+
+initRefresh();
diff --git a/airflow/www/templates/airflow/graph.html b/airflow/www/templates/airflow/graph.html
index da03b89..9ef79f3 100644
--- a/airflow/www/templates/airflow/graph.html
+++ b/airflow/www/templates/airflow/graph.html
@@ -22,6 +22,14 @@
{% block page_title %}{{ dag.dag_id }} - Graph - {{ appbuilder.app_name }}{% endblock %}
+{% block head_meta %}
+ {{ super() }}
+ <meta name="dag_id" content="{{ dag.dag_id }}">
+ <meta name="execution_date" content="{{ execution_date }}">
+ <meta name="arrange" content="{{ arrange }}">
+ <meta name="task_instances_url" content="{{ url_for('Airflow.task_instances') }}">
+{% endblock %}
+
{% block head_css %}
{{ super() }}
<link rel="stylesheet" type="text/css" href="{{ url_for_asset('graph.css') }}">
@@ -116,706 +124,15 @@
{% block tail %}
{{ super() }}
+ <script>
+ const nodes = {{ nodes|tojson }};
+ const edges = {{ edges|tojson }};
+ const tasks = {{ tasks|tojson }};
+ let taskInstances = {{ task_instances|tojson }};
+ </script>
<script src="{{ url_for_asset('d3.min.js') }}"></script>
<script src="{{ url_for_asset('dagre-d3.min.js') }}"></script>
<script src="{{ url_for_asset('d3-shape.min.js') }}"></script>
<script src="{{ url_for_asset('d3-tip.js') }}"></script>
- <script src="{{ url_for_asset('taskInstances.js') }}"></script>
- <script>
- var nodes = {{ nodes|tojson }};
- var edges = {{ edges|tojson }};
- var execution_date = "{{ execution_date }}";
- var arrange = "{{ arrange }}";
- var task_group_tips = get_task_group_tips(nodes);
- // This maps the actual task_id to the current graph node id that contains the task
- // (because tasks may be grouped into a group node)
- var map_task_to_node = new Map()
-
- // Below variables are being used in dag.js
- var tasks = {{ tasks|tojson }};
- var task_instances = {{ task_instances|tojson }};
- var getTaskInstanceURL = "{{ url_for('Airflow.task_instances') }}" +
- "?dag_id=" + encodeURIComponent(dag_id) + "&execution_date=" +
- encodeURIComponent(execution_date);
-
- var duration = 500;
- var stateFocusMap = {
- 'success': false,
- 'running':false,
- 'failed':false,
- 'skipped': false,
- 'upstream_failed': false,
- 'up_for_reschedule': false,
- 'up_for_retry': false,
- 'queued': false,
- 'no_status': false
- };
- const taskTip = d3.tip()
- .attr('class', 'tooltip d3-tip')
- .html(function(toolTipHtml) {
- return toolTipHtml;
- });
-
- // Preparation of DagreD3 data structures
- // "compound" is set to true to make use of clusters to display TaskGroup.
- var g = new dagreD3.graphlib.Graph({compound: true}).setGraph({
- nodesep: 30,
- ranksep: 15,
- rankdir: arrange,
- })
- .setDefaultEdgeLabel(function() { return { lineInterpolate: 'basis' } });
-
- var render = dagreD3.render(),
- svg = d3.select("#graph-svg"),
- innerSvg = d3.select("#graph-svg g");
-
- // Update the page to show the latest DAG.
- function draw() {
- innerSvg.remove()
- innerSvg = svg.append("g")
- // Run the renderer. This is what draws the final graph.
- innerSvg.call(render, g);
- innerSvg.call(taskTip)
-
- // When an expanded group is clicked, collapse it.
- d3.selectAll("g.cluster").on("click", function (node_id) {
- if (d3.event.defaultPrevented) // Ignore dragging actions.
- return;
- node = g.node(node_id)
- collapse_group(node_id, node)
- })
- // When a node is clicked, action depends on the node type.
- d3.selectAll("g.node").on("click", function (node_id) {
- node = g.node(node_id)
- if (node.children != undefined && Object.keys(node.children).length > 0) {
- // A group node
- if (d3.event.defaultPrevented) // Ignore dragging actions.
- return;
- expand_group(node_id, node)
- } else if (node_id in tasks) {
- // A task node
- task = tasks[node_id];
- if (node_id in task_instances)
- try_number = task_instances[node_id].try_number;
- else
- try_number = 0;
-
- if (task.task_type == "SubDagOperator")
- call_modal(node_id, execution_date, task.extra_links, try_number, true);
- else
- call_modal(node_id, execution_date, task.extra_links, try_number, undefined);
- } else {
- // join node between TaskGroup. Ignore.
- }
- });
-
- d3.selectAll('g.node').on('mouseover', function (d) {
- d3.select(this).selectAll('rect').attr('data-highlight', 'highlight');
- highlightNodes(g.predecessors(d));
- highlightNodes(g.successors(d));
- adjacent_node_names = [d, ...g.predecessors(d), ...g.successors(d)];
-
- d3.selectAll("g.nodes g.node")
- .filter(x => !adjacent_node_names.includes(x))
- .attr('data-highlight', 'fade');
-
- d3.selectAll("g.edgePath")[0].forEach(function (x) {
- var val = g.nodeEdges(d).includes(x.__data__) ? 'highlight' : 'fade';
- d3.select(x).attr('data-highlight', val);
- });
- d3.selectAll('g.edgeLabel')[0].forEach(function (x) {
- if (!g.nodeEdges(d).includes(x.__data__)) {
- d3.select(x).attr('data-highlight', 'fade');
- }
- });
- });
-
- d3.selectAll('g.node').on('mouseout', function (d) {
- d3.select(this).selectAll('rect, circle').attr('data-highlight', null);
- unHighlightNodes(g.predecessors(d))
- unHighlightNodes(g.successors(d))
- d3.selectAll('g.node, g.edgePath, g.edgeLabel')
- .attr('data-highlight', null);
- localStorage.removeItem(focused_group_key(dag_id));
- });
- updateNodesStates(task_instances);
- setUpZoomSupport();
- }
-
- var zoom = null;
-
- function setUpZoomSupport() {
- // Set up zoom support for Graph
- zoom = d3.behavior.zoom().on("zoom", function() {
- innerSvg.attr("transform", `translate(${d3.event.translate})scale(${d3.event.scale})`);
- });
- svg.call(zoom);
-
- // Centering the DAG on load
- // Get Dagre Graph dimensions
- var graphWidth = g.graph().width;
- var graphHeight = g.graph().height;
- // Get SVG dimensions
- var padding = 20;
- var svgBb = svg.node().getBoundingClientRect();
- var width = svgBb.width - padding*2;
- var height = svgBb.height - padding; // we are not centering the dag vertically
-
- // Calculate applicable scale for zoom
- zoomScale = Math.min(
- Math.min(width / graphWidth, height / graphHeight),
- 1.5, // cap zoom level to 1.5 so nodes are not too large
- );
-
- zoom.translate([(width/2) - ((graphWidth*zoomScale)/2) + padding, padding]);
- zoom.scale(zoomScale);
- zoom.event(innerSvg);
- }
-
- function highlightNodes(nodes) {
- nodes.forEach (function (nodeid) {
- const my_node = g.node(nodeid).elem
- d3.select(my_node)
- .selectAll('rect, circle')
- .attr('data-highlight', 'highlight');
- });
- }
-
- function unHighlightNodes(nodes) {
- nodes.forEach (function (nodeid) {
- const my_node = g.node(nodeid).elem
- d3.select(my_node)
- .selectAll('rect, circle')
- .attr('data-highlight', null);
- });
- }
-
- d3.selectAll('.js-state-legend-item')
- .on('mouseover', function() {
- if (!stateIsSet()) {
- state = $(this).data('state');
- focusState(state);
- }
- })
- .on('mouseout', function() {
- if (!stateIsSet()) {
- clearFocus();
- }
- });
-
- d3.selectAll('.js-state-legend-item').on('click', function() {
- state = $(this).data('state');
-
- clearFocus();
- if (!stateFocusMap[state]) {
- color = d3.select(this).style('border-color');
- focusState(state, this, color);
- setFocusMap(state);
- } else {
- setFocusMap();
- d3.selectAll('.js-state-legend-item')
- .style('background-color', null);
- }
- });
-
- // Returns true if a node's id or its children's id matches search_text
- function node_matches(node_id, search_text) {
- if (node_id.indexOf(search_text) > -1)
- return true;
-
- // The node's own id does not match, it may have children that match
- var node = g.node(node_id)
- if (node.children != undefined) {
- var children = get_children_ids(node);
- for(const child of children) {
- if(child.indexOf(search_text) > -1)
- return true
- }
- }
- }
-
- d3.select("#searchbox").on("keyup", function() {
- var s = document.getElementById('searchbox').value;
-
- if (s == '') return;
-
- var match = null;
-
- if (stateIsSet()) {
- clearFocus();
- setFocusMap();
- }
-
- d3.selectAll('g.nodes g.node').filter(function(d, i){
- if (s == '') {
- d3.selectAll('g.edgePaths, g.edgeLabel').attr('data-highlight', null);
- d3.select(this).attr('data-highlight', null);
- } else {
- d3.selectAll('g.edgePaths, g.edgeLabel').attr('data-highlight', 'fade');
- if (node_matches(d, s)) {
- if (!match) match = this;
- d3.select(this).attr('data-highlight', null);
- } else {
- d3.select(this).attr('data-highlight', 'fade');
- }
- }
- });
-
- // This moves the matched node to the center of the graph area
- if(match) {
- var transform = d3.transform(d3.select(match).attr("transform"));
-
- var svgBb = svg.node().getBoundingClientRect();
- transform.translate = [
- svgBb.width / 2 - transform.translate[0],
- svgBb.height / 2 - transform.translate[1]
- ];
- transform.scale = [1, 1];
-
- if(zoom != null) {
- zoom.translate(transform.translate);
- zoom.scale(1);
- zoom.event(innerSvg);
- }
- }
- });
-
- function clearFocus() {
- d3.selectAll('g.node, g.edgePaths, g.edgeLabel')
- .attr('data-highlight', null);
- localStorage.removeItem(focused_group_key(dag_id));
- }
-
- function focusState(state, node, color) {
- d3.selectAll('g.node, g.edgePaths, g.edgeLabel')
- .attr('data-highlight', 'fade');
- d3.selectAll(`g.node.${state}`)
- .attr('data-highlight', null);
- d3.selectAll(`g.node.${state} rect`)
- .attr('data-highlight', null);
- d3.select(node)
- .style('background-color', color);
- }
-
- function setFocusMap(state) {
- for (var key in stateFocusMap) {
- stateFocusMap[key] = false;
- }
- if (state != null) {
- stateFocusMap[state] = true;
- }
- }
-
- function stateIsSet() {
- for (var key in stateFocusMap) {
- if (stateFocusMap[key]) {
- return true;
- }
- }
- return false;
- }
-
- function handleRefresh() {
- $('#loading-dots').css('display', 'inline-block');
- $.get(getTaskInstanceURL)
- .done(
- (tis) => {
- task_instances = JSON.parse(tis)
- updateNodesStates(task_instances);
- setTimeout(function() { $('#loading-dots').hide(); }, 500);
- $('#error').hide();
- }
- ).fail((_, textStatus, err) => {
- $('#error_msg').text(`${textStatus}: ${err}`);
- $('#error').show();
- setTimeout(function() { $('#loading-dots').hide(); }, 500);
- $('#chart_section').hide(1000);
- $('#datatable_section').hide(1000);
- });
- }
-
- var refreshInterval;
-
- function startOrStopRefresh() {
- if ($('#auto_refresh').is(':checked')) {
- refreshInterval = setInterval(function() {
- handleRefresh();
- }, 3000); // run refresh every 3 seconds
- } else {
- clearInterval(refreshInterval);
- }
- }
-
- $('#auto_refresh').change(function() {
- if ($('#auto_refresh').is(':checked')) {
- // Run an initial refesh before starting interval if manually turned on
- handleRefresh();
- localStorage.removeItem('disableAutoRefresh');
- } else {
- localStorage.setItem('disableAutoRefresh', 'true');
- }
- startOrStopRefresh();
- });
-
- function initRefresh() {
- if (!!localStorage.getItem('disableAutoRefresh')) {
- $('#auto_refresh').removeAttr('checked');
- }
- startOrStopRefresh();
- d3.select('#refresh_button').on('click', () => handleRefresh());
- }
-
- // Generate tooltip for a group node
- function group_tooltip(node_id, tis) {
- var num_map = new Map([["success", 0],
- ["failed", 0],
- ["upstream_failed", 0],
- ["up_for_retry", 0],
- ["running", 0],
- ["no_status", 0]]
- );
- for(const child of get_children_ids(g.node(node_id))) {
- if(child in tis) {
- const ti = tis[child];
- const state_key = ti.state == null ? "no_status" : ti.state;
- if(num_map.has(state_key))
- num_map.set(state_key, num_map.get(state_key) + 1);
- }
- }
-
- const tip = task_group_tips.get(node_id);
- let tt = `${escapeHtml(tip)}<br><br>`;
- for(const [key, val] of num_map.entries())
- tt += `<strong>${escapeHtml(key)}:</strong> ${val} <br>`;
-
- return tt;
- }
-
- // Build a map mapping node id to tooltip for all the TaskGroups.
- function get_task_group_tips(node) {
- var tips = new Map();
- if(node.children != undefined) {
- tips.set(node.id, node.tooltip);
-
- for(const child of node.children.values()) {
- for(const [key, val] of get_task_group_tips(child))
- tips.set(key, val);
- }
- }
- return tips;
- }
-
- // Assigning css classes based on state to nodes
- // Initiating the tooltips
- function updateNodesStates(tis) {
- for(const node_id of g.nodes())
- {
- elem = g.node(node_id).elem;
- elem.setAttribute("class", `node enter ${get_node_state(node_id, tis)}`);
- elem.setAttribute("data-toggle", "tooltip");
-
- const task_id = node_id;
- elem.onmouseover = (evt) => {
- if(task_id in tis) {
- const tt = tiTooltip(tis[task_id]);
- taskTip.show(tt, evt.target); // taskTip is defined in graph.html
- } else if(task_group_tips.has(task_id)) {
- const tt = group_tooltip(task_id, tis)
- taskTip.show(tt, evt.target);
- } else if (task_id in tasks) {
- const tt = taskNoInstanceTooltip(task_id, tasks[task_id]);
- taskTip.show(tt, evt.target)
- }
- };
- elem.onmouseout = taskTip.hide;
- elem.onclick = taskTip.hide;
- }
- }
-
-
- // Returns list of children id of the given task group
- function get_children_ids(group) {
- var children = []
- for(const [key, val] of Object.entries(group.children)) {
- if(val.children == undefined) {
- // node
- children.push(val.id)
- } else {
- // group
- const sub_group_children = get_children_ids(val)
- for(const id of sub_group_children) {
- children.push(id)
- }
- }
- }
- return children
- }
-
- // Return list of all task group ids in the given task group including the given group itself.
- function get_all_group_ids(group) {
- var children = [group.id];
-
- for (const [key, val] of Object.entries(group.children)) {
- if (val.children != undefined) {
- // group
- const sub_group_children = get_all_group_ids(val)
- for (const id of sub_group_children) {
- children.push(id);
- }
- }
- }
- return children;
- }
-
-
- // Return the state for the node based on the state of its taskinstance or that of its
- // children if it's a group node
- function get_node_state(node_id, tis) {
- node = g.node(node_id)
-
- if (node.children == undefined) {
- if (node_id in tis) {
- return tis[node_id].state || 'no_status';
- }
- return 'no_status';
- }
- var children = get_children_ids(node);
-
- children_states = new Set();
- children.forEach(function(task_id) {
- if (task_id in tis) {
- var state = tis[task_id].state;
- children_states.add(state == null ? "no_status" : state);
- }
- })
-
- // In this order, if any of these states appeared in children_states, return it as
- // the group state.
- var priority = ["failed", "upstream_failed", "up_for_retry","up_for_reschedule",
- "queued", "scheduled", "sensing", "running", "shutdown", "removed",
- "no_status", "success", "skipped"]
-
- for(const state of priority) {
- if (children_states.has(state))
- return state
- }
- return "no_status"
- }
-
- // Returns the key used to store expanded task group ids in localStorage
- function expanded_groups_key(dag_id) {
- return `expanded_groups_${dag_id}`;
- }
-
- // Returns the key used to store the focused task group id in localStorage
- function focused_group_key(dag_id) {
- return `focused_group_${dag_id}`;
- }
-
- // Focus the graph on the expanded/collapsed node
- function focus_group(node_id) {
- if(node_id != null && zoom != null) {
- const x = g.node(node_id).x;
- const y = g.node(node_id).y;
- // This is the total canvas size.
- const svg_box = svg.node().getBoundingClientRect();
- const width = svg_box.width;
- const height = svg_box.height;
-
- // This is the size of the node or the cluster (i.e. group)
- var rect = d3.selectAll("g.node").filter(x => {return x == node_id}).select('rect');
- if (rect.empty())
- rect = d3.selectAll("g.cluster").filter(x => {return x == node_id}).select('rect');
-
- // Is there a better way to get node_width and node_height ?
- const [node_width, node_height] = [rect[0][0].attributes.width.value, rect[0][0].attributes.height.value];
-
- // Calculate zoom scale to fill most of the canvas with the node/cluster in focus.
- const scale = Math.min(
- Math.min(width / node_width, height / node_height),
- 1.5, // cap zoom level to 1.5 so nodes are not too large
- ) * 0.9;
-
- var [delta_x, delta_y] = [width / 2 - x * scale, height / 2 - y * scale];
- zoom.translate([delta_x, delta_y]);
- zoom.scale(scale);
- zoom.event(innerSvg.transition().duration(duration));
-
- const children = new Set(g.children(node_id))
- // Set data attr to highlight the focused group (via CSS).
- d3.selectAll('g.nodes g.node').filter(function(d, i){
- if (d == node_id || children.has(d)) {
- d3.select(this)
- .attr('data-highlight', null);
- } else {
- d3.select(this)
- .attr('data-highlight', 'fade');
- }
- });
-
- localStorage.setItem(focused_group_key(dag_id), node_id);
- }
- }
-
- // Expands a group node
- function expand_group(node_id, node, focus=true) {
- node.children.forEach(function (val) {
- // Set children nodes
- g.setNode(val.id, val.value)
- map_task_to_node.set(val.id, val.id)
- g.node(val.id).id = val.id
- if (val.children != undefined) {
- // Set children attribute so that the group can be expanded later when needed.
- group_node = g.node(val.id)
- group_node.children = val.children
- // Map task that are under this node to this node's id
- for(const child_id of get_children_ids(val))
- map_task_to_node.set(child_id, val.id)
- }
- // Only call setParent if node is not the root node.
- if (node_id != null)
- g.setParent(val.id, node_id)
- })
-
- // Add edges
- edges.forEach(function(edge) {
- source_id = map_task_to_node.get(edge.source_id)
- target_id = map_task_to_node.get(edge.target_id)
- if(source_id != target_id && !g.hasEdge(source_id, target_id))
- g.setEdge(source_id, target_id, {
- curve: d3.curveBasis,
- arrowheadClass: 'arrowhead',
- label: edge.label
- });
- })
-
- g.edges().forEach(function (edge) {
- // Remove edges that were associated with the expanded group node..
- if (node_id == edge.v || node_id == edge.w) {
- g.removeEdge(edge.v, edge.w);
- }
- });
-
- draw();
-
- if (focus) {
- focus_group(node_id);
- }
-
- save_expanded_group(node_id);
- }
-
- // Remove the node with this node_id from g.
- function remove_node(node_id) {
- if (g.hasNode(node_id)) {
- node = g.node(node_id)
- if(node.children != undefined) {
- // If the child is an expanded group node, remove children too.
- node.children.forEach(function (child) {
- remove_node(child.id);
- })
- }
- }
- g.removeNode(node_id);
- }
-
- // Collapse the children of the given group node.
- function collapse_group(node_id, node) {
- // Remove children nodes
- node.children.forEach(function(child) {
- remove_node(child.id)
- })
- // Map task that are under this node to this node's id
- for(const child_id of get_children_ids(node))
- map_task_to_node.set(child_id, node_id)
-
- node = g.node(node_id)
-
- // Set children edges onto the group edge
- edges.forEach(function(edge) {
- source_id = map_task_to_node.get(edge.source_id)
- target_id = map_task_to_node.get(edge.target_id)
- if (source_id != target_id && !g.hasEdge(source_id, target_id)) {
- g.setEdge(source_id, target_id, {
- curve: d3.curveBasis,
- arrowheadClass: 'arrowhead',
- });
- }
- })
-
- draw()
- focus_group(node_id)
-
- remove_expanded_group(node_id, node);
- }
-
- function get_saved_groups(dag_id) {
- // expanded_groups is a Set
- try {
- var expanded_groups = new Set(JSON.parse(localStorage.getItem(expanded_groups_key(dag_id))));
- } catch {
- var expanded_groups = new Set();
- }
-
- return expanded_groups;
- }
-
- // Clean up invalid group_ids from saved_group_ids (e.g. due to DAG changes)
- function prune_invalid_saved_group_ids() {
- // All the group_ids in the whole DAG
- const all_group_ids = new Set(get_all_group_ids(nodes));
- var expanded_groups = get_saved_groups(dag_id);
- expanded_groups = Array.from(expanded_groups).filter(group_id => all_group_ids.has(group_id));
- localStorage.setItem(expanded_groups_key(dag_id), JSON.stringify(expanded_groups));
- }
-
- // Remember the expanded groups in local storage so that it can be used to restore the expanded state
- // of task groups.
- function save_expanded_group(node_id) {
- // expanded_groups is a Set
- var expanded_groups = get_saved_groups(dag_id);
- expanded_groups.add(node_id)
- localStorage.setItem(expanded_groups_key(dag_id), JSON.stringify(Array.from(expanded_groups)));
- }
-
- // Remove the node_id from the expanded state
- function remove_expanded_group(node_id, node) {
- var expanded_groups = get_saved_groups(dag_id);
- const child_group_ids = get_all_group_ids(node);
- child_group_ids.forEach(child_id => expanded_groups.delete(child_id));
- localStorage.setItem(expanded_groups_key(dag_id), JSON.stringify(Array.from(expanded_groups)));
- }
-
- // Restore previously expanded task groups
- function expand_saved_groups(expanded_groups, node) {
- if (node.children == undefined) {
- return;
- }
-
- node.children.forEach(function (child_node) {
- if(expanded_groups.has(child_node.id)) {
- expand_group(child_node.id, g.node(child_node.id), false);
-
- expand_saved_groups(expanded_groups, child_node);
- }
- });
- }
-
- prune_invalid_saved_group_ids();
- const focus_node_id = localStorage.getItem(focused_group_key(dag_id));
- const expanded_groups = get_saved_groups(dag_id);
-
- // Always expand the root node
- expand_group(null, nodes);
-
- // Expand the node that were previously expanded
- expand_saved_groups(expanded_groups, nodes);
-
- // Restore focus (if available)
- if(g.hasNode(focus_node_id)) {
- focus_group(focus_node_id);
- }
-
- initRefresh();
- </script>
+ <script src="{{ url_for_asset('graph.js') }}"></script>
{% endblock %}
diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js
index 71dc63b..6c2dd32 100644
--- a/airflow/www/webpack.config.js
+++ b/airflow/www/webpack.config.js
@@ -43,7 +43,7 @@ const config = {
dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`],
flash: `${CSS_DIR}/flash.css`,
gantt: [`${CSS_DIR}/gantt.css`, `${JS_DIR}/gantt.js`],
- graph: `${CSS_DIR}/graph.css`,
+ graph: [`${CSS_DIR}/graph.css`, `${JS_DIR}/graph.js`],
ie: `${JS_DIR}/ie.js`,
loadingDots: `${CSS_DIR}/loading-dots.css`,
main: [`${CSS_DIR}/main.css`, `${JS_DIR}/main.js`],