You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/08/16 04:44:35 UTC
[airflow] branch v2-7-test updated (206699d059 -> b741d389ba)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit 206699d059 fixup! Update RELEASE_NOTES.rst
new 57114c6538 Improve graph nesting logic (#33421)
new b741d389ba fixup! Update RELEASE_NOTES.rst
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (206699d059)
\
N -- N -- N refs/heads/v2-7-test (b741d389ba)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
RELEASE_NOTES.rst | 1 +
airflow/www/static/js/components/Graph/Edge.tsx | 18 +----
airflow/www/static/js/dag/details/graph/Node.tsx | 1 -
airflow/www/static/js/dag/details/graph/index.tsx | 32 ++++-----
airflow/www/static/js/dag/details/graph/utils.ts | 81 ++++++++---------------
airflow/www/static/js/types/index.ts | 41 +++++++++++-
airflow/www/static/js/utils/graph.ts | 62 ++++++++++-------
7 files changed, 125 insertions(+), 111 deletions(-)
[airflow] 01/02: Improve graph nesting logic (#33421)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 57114c653891ac2cb5015a8f9e5011ebb9723220
Author: Brent Bovenzi <br...@astronomer.io>
AuthorDate: Wed Aug 16 00:38:46 2023 -0400
Improve graph nesting logic (#33421)
(cherry picked from commit 6547b8182f056b6d29445dec50cce3920d36f762)
---
airflow/www/static/js/components/Graph/Edge.tsx | 18 +----
airflow/www/static/js/dag/details/graph/Node.tsx | 1 -
airflow/www/static/js/dag/details/graph/index.tsx | 32 ++++-----
airflow/www/static/js/dag/details/graph/utils.ts | 81 ++++++++---------------
airflow/www/static/js/types/index.ts | 41 +++++++++++-
airflow/www/static/js/utils/graph.ts | 62 ++++++++++-------
6 files changed, 124 insertions(+), 111 deletions(-)
diff --git a/airflow/www/static/js/components/Graph/Edge.tsx b/airflow/www/static/js/components/Graph/Edge.tsx
index 910d2d3e43..3d9e144795 100644
--- a/airflow/www/static/js/components/Graph/Edge.tsx
+++ b/airflow/www/static/js/components/Graph/Edge.tsx
@@ -19,25 +19,13 @@
import React from "react";
import { Text, useTheme } from "@chakra-ui/react";
-import type { ElkEdgeSection, ElkLabel, ElkPoint, LayoutOptions } from "elkjs";
import { Group } from "@visx/group";
import { LinePath } from "@visx/shape";
+import type { EdgeData } from "src/types";
-interface EdgeProps {
- data?: {
- rest: {
- isSelected: boolean;
- sources: string[];
- targets: string[];
- sections: ElkEdgeSection[];
- junctionPoints?: ElkPoint[];
- id: string;
- labels?: ElkLabel[];
- layoutOptions?: LayoutOptions;
- isSetupTeardown?: boolean;
- };
- };
+export interface EdgeProps {
+ data?: EdgeData;
}
const CustomEdge = ({ data }: EdgeProps) => {
diff --git a/airflow/www/static/js/dag/details/graph/Node.tsx b/airflow/www/static/js/dag/details/graph/Node.tsx
index f2d6f92c09..6ae8489f13 100644
--- a/airflow/www/static/js/dag/details/graph/Node.tsx
+++ b/airflow/www/static/js/dag/details/graph/Node.tsx
@@ -44,7 +44,6 @@ export interface CustomNodeProps {
isOpen?: boolean;
isActive?: boolean;
setupTeardownType?: "setup" | "teardown";
- fullParentNode?: string;
labelStyle?: string;
style?: string;
}
diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx
index 1bdfe5ece7..29e07d65aa 100644
--- a/airflow/www/static/js/dag/details/graph/index.tsx
+++ b/airflow/www/static/js/dag/details/graph/index.tsx
@@ -24,7 +24,6 @@ import ReactFlow, {
Controls,
Background,
MiniMap,
- Node as ReactFlowNode,
useReactFlow,
Panel,
} from "reactflow";
@@ -35,7 +34,7 @@ import { useOffsetTop } from "src/utils";
import { useGraphLayout } from "src/utils/graph";
import Edge from "src/components/Graph/Edge";
-import Node, { CustomNodeProps } from "./Node";
+import Node from "./Node";
import { buildEdges, nodeStrokeColor, nodeColor, flattenNodes } from "./utils";
const nodeTypes = { custom: Node };
@@ -72,19 +71,17 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => {
const latestDagRunId = dagRuns[dagRuns.length - 1]?.runId;
const offsetTop = useOffsetTop(graphRef);
- const nodes: ReactFlowNode<CustomNodeProps>[] = useMemo(
+ const { nodes, edges: nodeEdges } = useMemo(
() =>
- graphData?.children
- ? flattenNodes({
- children: graphData.children,
- selected,
- openGroupIds,
- onToggleGroups,
- latestDagRunId,
- groups,
- hoveredTaskState,
- })
- : [],
+ flattenNodes({
+ children: graphData?.children,
+ selected,
+ openGroupIds,
+ onToggleGroups,
+ latestDagRunId,
+ groups,
+ hoveredTaskState,
+ }),
[
graphData?.children,
selected,
@@ -110,8 +107,13 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => {
setHasRendered(true);
}, [fitView, hasRendered, selected.taskId, getZoom]);
+ // merge & dedupe edges
+ const flatEdges = [...(graphData?.edges || []), ...(nodeEdges || [])].filter(
+ (value, index, self) => index === self.findIndex((t) => t.id === value.id)
+ );
+
const edges = buildEdges({
- edges: graphData?.edges,
+ edges: flatEdges,
nodes,
selectedTaskId: selected.taskId,
});
diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts
index aef01e5d53..3d85053619 100644
--- a/airflow/www/static/js/dag/details/graph/utils.ts
+++ b/airflow/www/static/js/dag/details/graph/utils.ts
@@ -28,7 +28,7 @@ import type { Task, TaskInstance, NodeType } from "src/types";
import type { CustomNodeProps } from "./Node";
interface FlattenNodesProps {
- children: NodeType[];
+ children?: NodeType[];
selected: SelectionProps;
groups: Task;
latestDagRunId: string;
@@ -50,15 +50,10 @@ export const flattenNodes = ({
hoveredTaskState,
}: FlattenNodesProps) => {
let nodes: ReactFlowNode<CustomNodeProps>[] = [];
+ let edges: ElkExtendedEdge[] = [];
+ if (!children) return { nodes, edges };
const parentNode = parent ? { parentNode: parent.id } : undefined;
- let fullParentNode: string | undefined;
- if (parent) {
- fullParentNode =
- parent.parentNode && !parent.id.startsWith(parent.parentNode)
- ? `${parent.parentNode}.${parent.id}`
- : parent.id;
- }
children.forEach((node) => {
let instance: TaskInstance | undefined;
const group = getTask({ taskId: node.id, task: groups });
@@ -90,7 +85,6 @@ export const flattenNodes = ({
}
onToggleGroups(newGroupIds);
},
- fullParentNode,
...node.value,
},
type: "custom",
@@ -105,10 +99,14 @@ export const flattenNodes = ({
...parentNode,
};
+ if (node.edges) {
+ edges = [...edges, ...node.edges];
+ }
+
nodes.push(newNode);
if (node.children) {
- const childNodes = flattenNodes({
+ const { nodes: childNodes, edges: childEdges } = flattenNodes({
children: node.children,
selected,
groups,
@@ -119,9 +117,13 @@ export const flattenNodes = ({
hoveredTaskState,
});
nodes = [...nodes, ...childNodes];
+ edges = [...edges, ...childEdges];
}
});
- return nodes;
+ return {
+ nodes,
+ edges,
+ };
};
export const nodeColor = ({
@@ -143,8 +145,12 @@ export const nodeStrokeColor = (
colors: Record<string, string>
) => (isSelected ? colors.blue[500] : "");
+interface Edge extends ElkExtendedEdge {
+ parentNode?: string;
+}
+
interface BuildEdgesProps {
- edges?: ElkExtendedEdge[];
+ edges?: Edge[];
nodes: ReactFlowNode<CustomNodeProps>[];
selectedTaskId?: string | null;
}
@@ -164,57 +170,24 @@ export const buildEdges = ({
type: "custom",
}))
.map((e) => {
- const sourceNode = nodes.find((n) => n.id === e.source);
- const targetNode = nodes.find((n) => n.id === e.target);
-
- // Before finding the depth of the edge, append the parentNode in case prefix_group_id is false
- const sourceIds = (
- sourceNode?.data.fullParentNode &&
- e.source.startsWith(sourceNode.data.fullParentNode)
- ? e.source
- : `${sourceNode?.data.fullParentNode}.${e.source}`
- ).split(".");
- const targetIds = (
- targetNode?.data.fullParentNode &&
- e.target.startsWith(targetNode.data.fullParentNode)
- ? e.target
- : `${targetNode?.data.fullParentNode}.${e.target}`
- ).split(".");
-
const isSelected =
selectedTaskId &&
(e.source === selectedTaskId || e.target === selectedTaskId);
- if (
- sourceIds.length === targetIds.length &&
- sourceIds[0] === targetIds[0]
- ) {
- let parentIds =
- sourceIds.length > targetIds.length ? sourceIds : targetIds;
-
- if (e.target.endsWith("_join_id") && e.source.endsWith("_join_id")) {
- /** edges between join ids are positioned absolutely,
- * other edges are positioned relative to their parent */
- parentIds = [];
- } else {
- // remove the last node
- parentIds.pop();
- }
- let parentX = 0;
- let parentY = 0;
-
- nodes
- .filter((n) => parentIds.some((p) => p === n.data.label))
- .forEach((p) => {
- parentX += p.position.x;
- parentY += p.position.y;
- });
-
+ if (e.data.rest?.parentNode) {
+ const parentNode = nodes.find((n) => n.id === e.data.rest.parentNode);
+ const parentX =
+ parentNode?.positionAbsolute?.x || parentNode?.position.x || 0;
+ const parentY =
+ parentNode?.positionAbsolute?.y || parentNode?.position.y || 0;
return {
...e,
data: {
rest: {
...e.data.rest,
+ labels: e.data.rest.labels?.map((l) =>
+ l.x && l.y ? { ...l, x: l.x + parentX, y: l.y + parentY } : l
+ ),
isSelected,
sections: e.data.rest.sections.map((s) => ({
...s,
diff --git a/airflow/www/static/js/types/index.ts b/airflow/www/static/js/types/index.ts
index 478b04d4f6..fc3634d9eb 100644
--- a/airflow/www/static/js/types/index.ts
+++ b/airflow/www/static/js/types/index.ts
@@ -18,7 +18,14 @@
*/
import type { CamelCase } from "type-fest";
-import type { ElkShape } from "elkjs";
+import type {
+ ElkExtendedEdge,
+ ElkEdgeSection,
+ ElkLabel,
+ ElkPoint,
+ ElkShape,
+ LayoutOptions,
+} from "elkjs";
import type * as API from "./api-generated";
type RunState = "success" | "running" | "queued" | "failed";
@@ -109,6 +116,20 @@ type RunOrdering = (
| "dataIntervalEnd"
)[];
+export interface MidEdge {
+ id: string;
+ sources: string[];
+ targets: string[];
+ isSetupTeardown: boolean | undefined;
+ parentNode: string | undefined;
+ labels: {
+ id: string;
+ text: string;
+ height: number;
+ width: number;
+ }[];
+}
+
interface DepNode {
id: string;
value: {
@@ -125,6 +146,7 @@ interface DepNode {
setupTeardownType?: "setup" | "teardown";
};
children?: DepNode[];
+ edges?: MidEdge[];
}
interface DepEdge {
@@ -132,9 +154,25 @@ interface DepEdge {
target: string;
}
+export interface EdgeData {
+ rest: {
+ isSelected: boolean;
+ sources: string[];
+ targets: string[];
+ sections: ElkEdgeSection[];
+ junctionPoints?: ElkPoint[];
+ id: string;
+ labels?: ElkLabel[];
+ layoutOptions?: LayoutOptions;
+ isSetupTeardown?: boolean;
+ parentNode?: string;
+ };
+}
+
export interface NodeType extends ElkShape {
value: DepNode["value"];
children?: NodeType[];
+ edges?: ElkExtendedEdge[];
}
export interface WebserverEdge {
@@ -142,6 +180,7 @@ export interface WebserverEdge {
sourceId: string;
targetId: string;
isSetupTeardown?: boolean;
+ parentNode?: string;
}
interface DatasetListItem extends API.Dataset {
diff --git a/airflow/www/static/js/utils/graph.ts b/airflow/www/static/js/utils/graph.ts
index cd2af9cf4c..08d5415833 100644
--- a/airflow/www/static/js/utils/graph.ts
+++ b/airflow/www/static/js/utils/graph.ts
@@ -68,6 +68,24 @@ const getDirection = (arrange: string) => {
}
};
+const formatEdge = (e: WebserverEdge, font: string, node?: DepNode) => ({
+ id: `${e.sourceId}-${e.targetId}`,
+ sources: [e.sourceId],
+ targets: [e.targetId],
+ isSetupTeardown: e.isSetupTeardown,
+ parentNode: node?.id,
+ labels: e.label
+ ? [
+ {
+ id: e.label,
+ text: e.label,
+ height: 16,
+ width: getTextWidth(e.label, font),
+ },
+ ]
+ : [],
+});
+
const generateGraph = ({
nodes,
edges: unformattedEdges,
@@ -102,6 +120,7 @@ const generateGraph = ({
const isOpen = openGroupIds?.includes(value.label);
const childCount =
children?.filter((c: DepNode) => !c.id.includes("join_id")).length || 0;
+ const childIds = children?.length ? getNestedChildIds(children) : [];
if (isOpen && children?.length) {
return {
...node,
@@ -116,11 +135,26 @@ const generateGraph = ({
"elk.padding": "[top=60,left=10,bottom=10,right=10]",
},
children: children.map(formatChildNode),
+ edges: filteredEdges
+ .filter((e) => {
+ if (
+ childIds.indexOf(e.sourceId) > -1 &&
+ childIds.indexOf(e.targetId) > -1
+ ) {
+ // Remove edge from array when we add it here
+ filteredEdges = filteredEdges.filter(
+ (fe) =>
+ !(fe.sourceId === e.sourceId && fe.targetId === e.targetId)
+ );
+ return true;
+ }
+ return false;
+ })
+ .map((e) => formatEdge(e, font, node)),
};
}
const isJoinNode = id.includes("join_id");
if (!isOpen && children?.length) {
- const childIds = getNestedChildIds(children);
filteredEdges = filteredEdges
// Filter out internal group edges
.filter(
@@ -153,30 +187,7 @@ const generateGraph = ({
const children = nodes.map(formatChildNode);
- const edges = filteredEdges
- .filter(
- (value, index, self) =>
- index ===
- self.findIndex(
- (t) => t.sourceId === value.sourceId && t.targetId === value.targetId
- )
- )
- .map((e) => ({
- id: `${e.sourceId}-${e.targetId}`,
- sources: [e.sourceId],
- targets: [e.targetId],
- isSetupTeardown: e.isSetupTeardown,
- labels: e.label
- ? [
- {
- id: e.label,
- text: e.label,
- height: 16,
- width: getTextWidth(e.label, font),
- },
- ]
- : [],
- }));
+ const edges = filteredEdges.map((e) => formatEdge(e, font));
return {
id: "root",
@@ -184,6 +195,7 @@ const generateGraph = ({
hierarchyHandling: "INCLUDE_CHILDREN",
"elk.direction": getDirection(arrange),
"spacing.edgeLabel": "10.0",
+ "elk.core.options.EdgeLabelPlacement": "CENTER",
},
children,
edges,
[airflow] 02/02: fixup! Update RELEASE_NOTES.rst
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b741d389ba0830d3d83913386101dc09484e37ae
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Mon Aug 14 12:16:57 2023 +0100
fixup! Update RELEASE_NOTES.rst
---
RELEASE_NOTES.rst | 17 ++++++++++++++---
newsfragments/33351.significant.rst | 6 ------
2 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index 6e0140da2f..f351d263d9 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -37,6 +37,13 @@ Old Graph View is removed (#32958)
""""""""""""""""""""""""""""""""""
The old Graph View is removed. The new Graph View is the default view now.
+The trigger UI form is skipped in web UI with 2.7.0 if no parameters are defined in a DAG (#33351)
+""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+If you are using ``dag_run.conf`` dictionary and web UI JSON entry to run your DAG you should either:
+
+* `Add params to your DAG <https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/params.html#use-params-to-provide-a-trigger-ui-form>`_
+* Enable the new configuration ``show_trigger_form_if_no_params`` to bring back old behaviour
The "db init", "db upgrade" commands and "[database] load_default_connections" configuration options are deprecated (#33136).
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
@@ -141,9 +148,8 @@ New Features
- Allows to choose SSL context for SMTP connection (#33070)
- New gantt tab (#31806)
- Load plugins from providers (#32692)
-- Add ``ExternalBranchPythonOperator`` (#32787)
+- Add ``BranchExternalPythonOperator`` (#32787, #33360)
- Add option for storing configuration description in providers (#32629)
-- Add an option to use a direct DB connection in KEDA when ``pgbouncer`` is enabled (#32608)
- Introduce Heartbeat Parameter to Allow ``Per-LocalTaskJob`` Configuration (#32313)
- Add Executors discovery and documentation (#32532)
- Add JobState for job state constants (#32549)
@@ -175,6 +181,7 @@ New Features
Improvements
""""""""""""
+- Improve graph nesting logic (#33421)
- Configurable health check threshold for triggerer (#33089, #33084)
- add dag_run_ids and task_ids filter for the batch task instance API endpoint (#32705)
- Ensure DAG-level references are filled on unmap (#33083)
@@ -194,7 +201,7 @@ Improvements
- Kubernetes Executor Load Time Optimizations (#30727)
- Save DAG parsing time if dag is not schedulable (#30911)
- Updates health check endpoint to include ``dag_processor`` status. (#32382)
-- Disable default allowing the testing of connections in UI, API and CLI (#32052)
+- Disable default allowing the testing of connections in UI, API and CLI (#32052, #33342)
- Fix config var types under the scheduler section (#32132)
- Allow to sort Grid View alphabetically (#32179)
- Add hostname to triggerer metric ``[triggers.running]`` (#32050)
@@ -226,6 +233,7 @@ Improvements
Bug Fixes
"""""""""
+- Remove user sessions when resetting password (#33347)
- ``Gantt chart:`` Use earliest/oldest ti dates if different than dag run start/end (#33215)
- Fix ``virtualenv`` detection for Python ``virtualenv`` operator (#33223)
- Correctly log when there are problems trying to ``chmod`` ``airflow.cfg`` (#33118)
@@ -281,6 +289,7 @@ Bug Fixes
Misc/Internal
"""""""""""""
+- Modify pathspec version restriction (#33349)
- Refactor: Simplify code in ``dag_processing`` (#33161)
- For now limit ``Pydantic`` to ``< 2.0.0`` (#33235)
- Refactor: Simplify code in models (#33181)
@@ -332,6 +341,8 @@ Misc/Internal
Docs only changes
"""""""""""""""""
+- Fix links in security docs (#33329)
+- Clarify UI user types in security model (#33021)
- Add links to ``DAGRun / DAG / Task`` in templates-ref.rst (#33013)
- Add docs of how to test for DAG Import Errors (#32811)
- Clean-up of our new security page (#32951)
diff --git a/newsfragments/33351.significant.rst b/newsfragments/33351.significant.rst
deleted file mode 100644
index d8f91c4c20..0000000000
--- a/newsfragments/33351.significant.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-The trigger UI form is skipped in web UI with 2.7.0 if no parameters are defined in a DAG.
-
-If you are using ``dag_run.conf`` dictionary and web UI JSON entry to run your DAG you should either:
-
-* `Add params to your DAG <https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/params.html#use-params-to-provide-a-trigger-ui-form>`_
-* Enable the new configuration ``show_trigger_form_if_no_params`` to bring back old behaviour