You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2024/02/22 18:26:46 UTC
(airflow) branch main updated: Show dataset events above task/run details in grid view (#37603)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 3f8c0b9180 Show dataset events above task/run details in grid view (#37603)
3f8c0b9180 is described below
commit 3f8c0b9180a043f453576f4d571af8b3fc2f97b9
Author: Brent Bovenzi <br...@astronomer.io>
AuthorDate: Thu Feb 22 13:26:38 2024 -0500
Show dataset events above task/run details in grid view (#37603)
* Show dataset events above task/run details in grid view
* Add extra to dataset event tables
---
.../www/static/js/api/useUpstreamDatasetEvents.ts | 2 +-
.../js/dag/details/dagRun/DatasetTriggerEvents.tsx | 13 +-
.../dag/details/dagRun/{index.tsx => Details.tsx} | 49 ++----
airflow/www/static/js/dag/details/dagRun/index.tsx | 175 ++-------------------
.../details/taskInstance/DatasetUpdateEvents.tsx | 15 +-
.../static/js/dag/details/taskInstance/Details.tsx | 44 +-----
.../js/dag/details/taskInstance/TriggererInfo.tsx | 65 ++++++++
.../static/js/dag/details/taskInstance/index.tsx | 6 +
airflow/www/static/js/datasets/DatasetEvents.tsx | 7 +
9 files changed, 127 insertions(+), 249 deletions(-)
diff --git a/airflow/www/static/js/api/useUpstreamDatasetEvents.ts b/airflow/www/static/js/api/useUpstreamDatasetEvents.ts
index 995941613c..5ba492183c 100644
--- a/airflow/www/static/js/api/useUpstreamDatasetEvents.ts
+++ b/airflow/www/static/js/api/useUpstreamDatasetEvents.ts
@@ -33,7 +33,7 @@ export default function useUpstreamDatasetEvents({ runId }: Props) {
const upstreamEventsUrl = (
getMetaValue("upstream_dataset_events_api") ||
`api/v1/dags/${dagId}/dagRuns/_DAG_RUN_ID_/upstreamDatasetEvents`
- ).replace("_DAG_RUN_ID_", runId);
+ ).replace("_DAG_RUN_ID_", encodeURIComponent(runId));
return axios.get<AxiosResponse, API.DatasetEventCollection>(
upstreamEventsUrl
);
diff --git a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx
index 07649d54d9..4e0ed4a956 100644
--- a/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx
+++ b/airflow/www/static/js/dag/details/dagRun/DatasetTriggerEvents.tsx
@@ -17,9 +17,10 @@
* under the License.
*/
import React, { useMemo } from "react";
-import { Box, Heading, Text } from "@chakra-ui/react";
+import { Box, Text } from "@chakra-ui/react";
import {
+ CodeCell,
DatasetLink,
Table,
TaskInstanceLink,
@@ -55,6 +56,12 @@ const DatasetTriggerEvents = ({ runId }: Props) => {
accessor: "timestamp",
Cell: TimeCell,
},
+ {
+ Header: "Extra",
+ accessor: "extra",
+ Cell: CodeCell,
+ disableSortBy: true,
+ },
],
[]
);
@@ -63,7 +70,9 @@ const DatasetTriggerEvents = ({ runId }: Props) => {
return (
<Box mt={3} flexGrow={1}>
- <Heading size="md">Dataset Events</Heading>
+ <Text as="strong" mb={3}>
+ Dataset Events
+ </Text>
<Text>Dataset updates that triggered this DAG run.</Text>
<Table data={data} columns={columns} isLoading={isLoading} />
</Box>
diff --git a/airflow/www/static/js/dag/details/dagRun/index.tsx b/airflow/www/static/js/dag/details/dagRun/Details.tsx
similarity index 81%
copy from airflow/www/static/js/dag/details/dagRun/index.tsx
copy to airflow/www/static/js/dag/details/dagRun/Details.tsx
index 2bbdc5cdb0..c769da72d5 100644
--- a/airflow/www/static/js/dag/details/dagRun/index.tsx
+++ b/airflow/www/static/js/dag/details/dagRun/Details.tsx
@@ -16,38 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-import React, { useEffect, useRef } from "react";
+import React, { useEffect } from "react";
import {
Flex,
Box,
Button,
- Divider,
Spacer,
Table,
Tbody,
Tr,
Td,
useClipboard,
+ Text,
} from "@chakra-ui/react";
import ReactJson from "react-json-view";
-import { useGridData } from "src/api";
-import { getMetaValue, useOffsetTop } from "src/utils";
import type { DagRun as DagRunType } from "src/types";
import { SimpleStatus } from "src/dag/StatusBox";
import { ClipboardText } from "src/components/Clipboard";
import { formatDuration, getDuration } from "src/datetime_utils";
import Time from "src/components/Time";
import RunTypeIcon from "src/components/RunTypeIcon";
-import NotesAccordion from "src/dag/details/NotesAccordion";
-
-import DatasetTriggerEvents from "./DatasetTriggerEvents";
-
-const dagId = getMetaValue("dag_id");
interface Props {
- runId: DagRunType["runId"];
+ run: DagRunType;
}
const formatConf = (conf: string | null | undefined): string => {
@@ -57,14 +50,7 @@ const formatConf = (conf: string | null | undefined): string => {
return JSON.stringify(JSON.parse(conf), null, 4);
};
-const DagRun = ({ runId }: Props) => {
- const {
- data: { dagRuns },
- } = useGridData();
- const detailsRef = useRef<HTMLDivElement>(null);
- const offsetTop = useOffsetTop(detailsRef);
-
- const run = dagRuns.find((dr) => dr.runId === runId);
+const DagRunDetails = ({ run }: Props) => {
const { onCopy, setValue, hasCopied } = useClipboard(formatConf(run?.conf));
useEffect(() => {
@@ -84,25 +70,13 @@ const DagRun = ({ runId }: Props) => {
externalTrigger,
conf,
confIsJson,
- note,
} = run;
return (
- <Box
- maxHeight={`calc(100% - ${offsetTop}px)`}
- ref={detailsRef}
- overflowY="auto"
- pb={4}
- >
- <Box px={4}>
- <NotesAccordion
- dagId={dagId}
- runId={runId}
- initialValue={note}
- key={dagId + runId}
- />
- </Box>
- <Divider my={0} />
+ <Box mt={3} flexGrow={1}>
+ <Text as="strong" mb={3}>
+ Dag Run Details
+ </Text>
<Table variant="striped">
<Tbody>
<Tr>
@@ -117,7 +91,7 @@ const DagRun = ({ runId }: Props) => {
<Tr>
<Td>Run ID</Td>
<Td>
- <ClipboardText value={runId} />
+ <ClipboardText value={run.runId} />
</Td>
</Tr>
<Tr>
@@ -212,11 +186,8 @@ const DagRun = ({ runId }: Props) => {
</Tr>
</Tbody>
</Table>
- {runType === "dataset_triggered" && (
- <DatasetTriggerEvents runId={runId} />
- )}
</Box>
);
};
-export default DagRun;
+export default DagRunDetails;
diff --git a/airflow/www/static/js/dag/details/dagRun/index.tsx b/airflow/www/static/js/dag/details/dagRun/index.tsx
index 2bbdc5cdb0..05679a598f 100644
--- a/airflow/www/static/js/dag/details/dagRun/index.tsx
+++ b/airflow/www/static/js/dag/details/dagRun/index.tsx
@@ -16,33 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-import React, { useEffect, useRef } from "react";
-import {
- Flex,
- Box,
- Button,
- Divider,
- Spacer,
- Table,
- Tbody,
- Tr,
- Td,
- useClipboard,
-} from "@chakra-ui/react";
-
-import ReactJson from "react-json-view";
+import React, { useRef } from "react";
+import { Box } from "@chakra-ui/react";
import { useGridData } from "src/api";
import { getMetaValue, useOffsetTop } from "src/utils";
import type { DagRun as DagRunType } from "src/types";
-import { SimpleStatus } from "src/dag/StatusBox";
-import { ClipboardText } from "src/components/Clipboard";
-import { formatDuration, getDuration } from "src/datetime_utils";
-import Time from "src/components/Time";
-import RunTypeIcon from "src/components/RunTypeIcon";
import NotesAccordion from "src/dag/details/NotesAccordion";
import DatasetTriggerEvents from "./DatasetTriggerEvents";
+import DagRunDetails from "./Details";
const dagId = getMetaValue("dag_id");
@@ -50,13 +33,6 @@ interface Props {
runId: DagRunType["runId"];
}
-const formatConf = (conf: string | null | undefined): string => {
- if (!conf) {
- return "";
- }
- return JSON.stringify(JSON.parse(conf), null, 4);
-};
-
const DagRun = ({ runId }: Props) => {
const {
data: { dagRuns },
@@ -65,27 +41,9 @@ const DagRun = ({ runId }: Props) => {
const offsetTop = useOffsetTop(detailsRef);
const run = dagRuns.find((dr) => dr.runId === runId);
- const { onCopy, setValue, hasCopied } = useClipboard(formatConf(run?.conf));
-
- useEffect(() => {
- setValue(formatConf(run?.conf));
- }, [run, setValue]);
if (!run) return null;
- const {
- state,
- runType,
- lastSchedulingDecision,
- dataIntervalStart,
- dataIntervalEnd,
- startDate,
- endDate,
- queuedAt,
- externalTrigger,
- conf,
- confIsJson,
- note,
- } = run;
+ const { runType, note } = run;
return (
<Box
@@ -94,127 +52,16 @@ const DagRun = ({ runId }: Props) => {
overflowY="auto"
pb={4}
>
- <Box px={4}>
- <NotesAccordion
- dagId={dagId}
- runId={runId}
- initialValue={note}
- key={dagId + runId}
- />
- </Box>
- <Divider my={0} />
- <Table variant="striped">
- <Tbody>
- <Tr>
- <Td>Status</Td>
- <Td>
- <Flex>
- <SimpleStatus state={state} mx={2} />
- {state || "no status"}
- </Flex>
- </Td>
- </Tr>
- <Tr>
- <Td>Run ID</Td>
- <Td>
- <ClipboardText value={runId} />
- </Td>
- </Tr>
- <Tr>
- <Td>Run type</Td>
- <Td>
- <RunTypeIcon runType={runType} />
- {runType}
- </Td>
- </Tr>
- {startDate && (
- <Tr>
- <Td>Run duration</Td>
- <Td>{formatDuration(getDuration(startDate, endDate))}</Td>
- </Tr>
- )}
- {lastSchedulingDecision && (
- <Tr>
- <Td>Last scheduling decision</Td>
- <Td>
- <Time dateTime={lastSchedulingDecision} />
- </Td>
- </Tr>
- )}
- {queuedAt && (
- <Tr>
- <Td>Queued at</Td>
- <Td>
- <Time dateTime={queuedAt} />
- </Td>
- </Tr>
- )}
- {startDate && (
- <Tr>
- <Td>Started</Td>
- <Td>
- <Time dateTime={startDate} />
- </Td>
- </Tr>
- )}
- {endDate && (
- <Tr>
- <Td>Ended</Td>
- <Td>
- <Time dateTime={endDate} />
- </Td>
- </Tr>
- )}
- {dataIntervalStart && dataIntervalEnd && (
- <>
- <Tr>
- <Td>Data interval start</Td>
- <Td>
- <Time dateTime={dataIntervalStart} />
- </Td>
- </Tr>
- <Tr>
- <Td>Data interval end</Td>
- <Td>
- <Time dateTime={dataIntervalEnd} />
- </Td>
- </Tr>
- </>
- )}
- <Tr>
- <Td>Externally triggered</Td>
- <Td>{externalTrigger ? "True" : "False"}</Td>
- </Tr>
- <Tr>
- <Td>Run config</Td>
- {confIsJson ? (
- <Td>
- <Flex>
- <ReactJson
- src={JSON.parse(conf ?? "")}
- name={false}
- theme="rjv-default"
- iconStyle="triangle"
- indentWidth={2}
- displayDataTypes={false}
- enableClipboard={false}
- style={{ backgroundColor: "inherit" }}
- />
- <Spacer />
- <Button aria-label="Copy" onClick={onCopy}>
- {hasCopied ? "Copied!" : "Copy"}
- </Button>
- </Flex>
- </Td>
- ) : (
- <Td>{conf ?? "None"}</Td>
- )}
- </Tr>
- </Tbody>
- </Table>
+ <NotesAccordion
+ dagId={dagId}
+ runId={runId}
+ initialValue={note}
+ key={dagId + runId}
+ />
{runType === "dataset_triggered" && (
<DatasetTriggerEvents runId={runId} />
)}
+ <DagRunDetails run={run} />
</Box>
);
};
diff --git a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx
index 919f4bca8a..ad1ecf9245 100644
--- a/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx
+++ b/airflow/www/static/js/dag/details/taskInstance/DatasetUpdateEvents.tsx
@@ -17,9 +17,10 @@
* under the License.
*/
import React, { useMemo } from "react";
-import { Box, Heading, Text } from "@chakra-ui/react";
+import { Box, Text } from "@chakra-ui/react";
import {
+ CodeCell,
DatasetLink,
Table,
TimeCell,
@@ -63,6 +64,12 @@ const DatasetUpdateEvents = ({ runId, taskId }: Props) => {
accessor: "createdDagruns",
Cell: TriggeredRuns,
},
+ {
+ Header: "Extra",
+ accessor: "extra",
+ Cell: CodeCell,
+ disableSortBy: true,
+ },
],
[]
);
@@ -70,8 +77,10 @@ const DatasetUpdateEvents = ({ runId, taskId }: Props) => {
const data = useMemo(() => datasetEvents, [datasetEvents]);
return (
- <Box mt={3} flexGrow={1}>
- <Heading size="md">Dataset Events</Heading>
+ <Box my={3} flexGrow={1}>
+ <Text as="strong" mb={3}>
+ Dataset Events
+ </Text>
<Text>Dataset updates caused by this task instance</Text>
<Table data={data} columns={columns} isLoading={isLoading} />
</Box>
diff --git a/airflow/www/static/js/dag/details/taskInstance/Details.tsx b/airflow/www/static/js/dag/details/taskInstance/Details.tsx
index 2fcbb74a57..5e78825e5c 100644
--- a/airflow/www/static/js/dag/details/taskInstance/Details.tsx
+++ b/airflow/www/static/js/dag/details/taskInstance/Details.tsx
@@ -18,7 +18,7 @@
*/
import React from "react";
-import { Text, Flex, Table, Tbody, Tr, Td, Code } from "@chakra-ui/react";
+import { Text, Flex, Table, Tbody, Tr, Td, Code, Box } from "@chakra-ui/react";
import { snakeCase } from "lodash";
import { getGroupAndMapSummary } from "src/utils";
@@ -32,7 +32,6 @@ import type {
TaskInstance as GridTaskInstance,
TaskState,
} from "src/types";
-import DatasetUpdateEvents from "./DatasetUpdateEvents";
interface Props {
gridInstance: GridTaskInstance;
@@ -48,7 +47,7 @@ const Details = ({ gridInstance, taskInstance, group }: Props) => {
const mappedStates = !taskInstance ? gridInstance.mappedStates : undefined;
- const { isMapped, tooltip, operator, hasOutletDatasets, triggerRule } = group;
+ const { isMapped, tooltip, operator, triggerRule } = group;
const { totalTasks, childTaskMap } = getGroupAndMapSummary({
group,
@@ -82,39 +81,7 @@ const Details = ({ gridInstance, taskInstance, group }: Props) => {
const isOverall = (isMapped || isGroup) && "Overall ";
return (
- <Flex flexWrap="wrap" justifyContent="space-between">
- {!!taskInstance?.trigger && !!taskInstance?.triggererJob && (
- <>
- <Text as="strong" mb={3}>
- Triggerer info
- </Text>
- <Table variant="striped" mb={3}>
- <Tbody>
- <Tr>
- <Td>Trigger class</Td>
- <Td>{`${taskInstance?.trigger?.classpath}`}</Td>
- </Tr>
- <Tr>
- <Td>Trigger ID</Td>
- <Td>{`${taskInstance?.trigger?.id}`}</Td>
- </Tr>
- <Tr>
- <Td>Trigger creation time</Td>
- <Td>{`${taskInstance?.trigger?.createdDate}`}</Td>
- </Tr>
- <Tr>
- <Td>Assigned triggerer</Td>
- <Td>{`${taskInstance?.triggererJob?.hostname}`}</Td>
- </Tr>
- <Tr>
- <Td>Latest triggerer heartbeat</Td>
- <Td>{`${taskInstance?.triggererJob?.latestHeartbeat}`}</Td>
- </Tr>
- </Tbody>
- </Table>
- </>
- )}
-
+ <Box mt={3} flexGrow={1}>
<Text as="strong" mb={3}>
Task Instance Details
</Text>
@@ -305,10 +272,7 @@ const Details = ({ gridInstance, taskInstance, group }: Props) => {
)}
</Tbody>
</Table>
- {hasOutletDatasets && (
- <DatasetUpdateEvents taskId={taskId} runId={runId} />
- )}
- </Flex>
+ </Box>
);
};
diff --git a/airflow/www/static/js/dag/details/taskInstance/TriggererInfo.tsx b/airflow/www/static/js/dag/details/taskInstance/TriggererInfo.tsx
new file mode 100644
index 0000000000..cb093d4337
--- /dev/null
+++ b/airflow/www/static/js/dag/details/taskInstance/TriggererInfo.tsx
@@ -0,0 +1,65 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from "react";
+import { Text, Table, Tbody, Tr, Td, Box } from "@chakra-ui/react";
+
+import type { API } from "src/types";
+
+interface Props {
+ taskInstance?: API.TaskInstance;
+}
+
+const TriggererInfo = ({ taskInstance }: Props) => {
+ if (!taskInstance?.trigger || !taskInstance?.triggererJob) return null;
+
+ return (
+ <Box mt={3} flexGrow={1}>
+ <Text as="strong" mb={3}>
+ Triggerer info
+ </Text>
+ <Table variant="striped" mb={3}>
+ <Tbody>
+ <Tr>
+ <Td>Trigger class</Td>
+ <Td>{`${taskInstance?.trigger?.classpath}`}</Td>
+ </Tr>
+ <Tr>
+ <Td>Trigger ID</Td>
+ <Td>{`${taskInstance?.trigger?.id}`}</Td>
+ </Tr>
+ <Tr>
+ <Td>Trigger creation time</Td>
+ <Td>{`${taskInstance?.trigger?.createdDate}`}</Td>
+ </Tr>
+ <Tr>
+ <Td>Assigned triggerer</Td>
+ <Td>{`${taskInstance?.triggererJob?.hostname}`}</Td>
+ </Tr>
+ <Tr>
+ <Td>Latest triggerer heartbeat</Td>
+ <Td>{`${taskInstance?.triggererJob?.latestHeartbeat}`}</Td>
+ </Tr>
+ </Tbody>
+ </Table>
+ </Box>
+ );
+};
+
+export default TriggererInfo;
diff --git a/airflow/www/static/js/dag/details/taskInstance/index.tsx b/airflow/www/static/js/dag/details/taskInstance/index.tsx
index 71f8717dd5..1b8f0a54ba 100644
--- a/airflow/www/static/js/dag/details/taskInstance/index.tsx
+++ b/airflow/www/static/js/dag/details/taskInstance/index.tsx
@@ -28,6 +28,8 @@ import NotesAccordion from "src/dag/details/NotesAccordion";
import TaskNav from "./Nav";
import ExtraLinks from "./ExtraLinks";
import Details from "./Details";
+import DatasetUpdateEvents from "./DatasetUpdateEvents";
+import TriggererInfo from "./TriggererInfo";
const dagId = getMetaValue("dag_id")!;
@@ -106,6 +108,10 @@ const TaskInstance = ({ taskId, runId, mapIndex }: Props) => {
tryNumber={taskInstance?.tryNumber || gridInstance.tryNumber}
/>
)}
+ {group.hasOutletDatasets && (
+ <DatasetUpdateEvents taskId={taskId} runId={runId} />
+ )}
+ <TriggererInfo taskInstance={taskInstance} />
<Details
gridInstance={gridInstance}
taskInstance={taskInstance}
diff --git a/airflow/www/static/js/datasets/DatasetEvents.tsx b/airflow/www/static/js/datasets/DatasetEvents.tsx
index dd200942c0..b58006ac81 100644
--- a/airflow/www/static/js/datasets/DatasetEvents.tsx
+++ b/airflow/www/static/js/datasets/DatasetEvents.tsx
@@ -27,6 +27,7 @@ import {
TimeCell,
TaskInstanceLink,
TriggeredRuns,
+ CodeCell,
} from "src/components/Table";
const Events = ({ datasetId }: { datasetId: number }) => {
@@ -66,6 +67,12 @@ const Events = ({ datasetId }: { datasetId: number }) => {
accessor: "createdDagruns",
Cell: TriggeredRuns,
},
+ {
+ Header: "Extra",
+ accessor: "extra",
+ Cell: CodeCell,
+ disableSortBy: true,
+ },
],
[]
);