You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2022/04/05 19:23:13 UTC
[airflow] branch reduce-tree-data-size updated (c338832ffa -> 0529654bb7)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a change to branch reduce-tree-data-size
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard c338832ffa Remove extraneous treeData fields
add a1fd82f2a5 Fix state and try number for failed mapped tasks (#22757)
new 0529654bb7 Remove extraneous treeData fields
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c338832ffa)
\
N -- N -- N refs/heads/reduce-tree-data-size (0529654bb7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airflow/www/utils.py | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
[airflow] 01/01: Remove extraneous treeData fields
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch reduce-tree-data-size
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0529654bb76c59aa8c2f1c91bb23bb811cae106e
Author: Brent Bovenzi <br...@gmail.com>
AuthorDate: Tue Apr 5 15:09:58 2022 -0400
Remove extraneous treeData fields
---
.../js/tree/details/content/dagRun/index.jsx | 2 +-
.../tree/details/content/taskInstance/Details.jsx | 21 +++++----
.../js/tree/details/content/taskInstance/Nav.jsx | 12 ++---
.../js/tree/details/content/taskInstance/index.jsx | 52 +++++++++++++---------
.../taskInstance/taskActions/MarkFailed.jsx | 2 +-
airflow/www/static/js/tree/treeDataUtils.js | 2 +-
airflow/www/utils.py | 7 ---
7 files changed, 50 insertions(+), 48 deletions(-)
diff --git a/airflow/www/static/js/tree/details/content/dagRun/index.jsx b/airflow/www/static/js/tree/details/content/dagRun/index.jsx
index 68e6d6526a..fd609b7ad0 100644
--- a/airflow/www/static/js/tree/details/content/dagRun/index.jsx
+++ b/airflow/www/static/js/tree/details/content/dagRun/index.jsx
@@ -37,6 +37,7 @@ import ClearRun from './ClearRun';
import { useTreeData } from '../../../api';
import { appendSearchParams, getMetaValue } from '../../../../utils';
+const dagId = getMetaValue('dag_id');
const graphUrl = getMetaValue('graph_url');
const dagRunDetailsUrl = getMetaValue('dagrun_details_url');
@@ -46,7 +47,6 @@ const DagRun = ({ runId }) => {
if (!run) return null;
const {
executionDate,
- dagId,
state,
runType,
duration,
diff --git a/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx b/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
index 44294f51f7..1ae907b00c 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
@@ -29,8 +29,8 @@ import { getDuration, formatDuration } from '../../../../datetime_utils';
import { SimpleStatus } from '../../../StatusBox';
import Time from '../../../Time';
-const Details = ({ instance, task }) => {
- const isGroup = !!task.children;
+const Details = ({ instance, group, operator }) => {
+ const isGroup = !!group.children;
const groupSummary = [];
const mapSummary = [];
@@ -38,16 +38,21 @@ const Details = ({ instance, task }) => {
taskId,
runId,
duration,
- operator,
startDate,
endDate,
state,
mappedStates,
} = instance;
+ const {
+ isMapped,
+ children,
+ tooltip,
+ } = group;
+
if (isGroup) {
const numMap = finalStatesMap();
- task.children.forEach((child) => {
+ children.forEach((child) => {
const taskInstance = child.instances.find((ti) => ti.runId === runId);
if (taskInstance) {
const stateKey = taskInstance.state == null ? 'no_status' : taskInstance.state;
@@ -68,7 +73,7 @@ const Details = ({ instance, task }) => {
});
}
- if (task.isMapped && mappedStates) {
+ if (isMapped && mappedStates) {
const numMap = finalStatesMap();
mappedStates.forEach((s) => {
const stateKey = s || 'no_status';
@@ -94,8 +99,8 @@ const Details = ({ instance, task }) => {
return (
<Flex flexWrap="wrap" justifyContent="space-between">
<Box>
- {task.tooltip && (
- <Text>{task.tooltip}</Text>
+ {tooltip && (
+ <Text>{tooltip}</Text>
)}
<Flex alignItems="center">
<Text as="strong">Status:</Text>
@@ -109,7 +114,7 @@ const Details = ({ instance, task }) => {
{groupSummary}
</>
)}
- {task.isMapped && (
+ {isMapped && (
<>
<br />
<Text as="strong">
diff --git a/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx b/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
index d0841d0119..17b423130a 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
@@ -27,6 +27,7 @@ import {
import { getMetaValue, appendSearchParams } from '../../../../utils';
+const dagId = getMetaValue('dag_id');
const isK8sExecutor = getMetaValue('k8s_or_k8scelery_executor') === 'True';
const numRuns = getMetaValue('num_runs');
const baseDate = getMetaValue('base_date');
@@ -40,14 +41,9 @@ const gridUrlNoRoot = getMetaValue('grid_url_no_root');
const LinkButton = ({ children, ...rest }) => (<Button as={Link} variant="ghost" colorScheme="blue" {...rest}>{children}</Button>);
-const Nav = ({ instance, isMapped }) => {
- const {
- taskId,
- dagId,
- operator,
- executionDate,
- } = instance;
-
+const Nav = ({
+ taskId, executionDate, operator, isMapped,
+}) => {
const params = new URLSearchParams({
task_id: taskId,
execution_date: executionDate,
diff --git a/airflow/www/static/js/tree/details/content/taskInstance/index.jsx b/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
index 1e787c7aba..93d22dbde7 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
@@ -34,8 +34,11 @@ import Logs from './Logs';
import TaskNav from './Nav';
import Details from './Details';
-import { useTreeData } from '../../../api';
+import { useTreeData, useTasks } from '../../../api';
import MappedInstances from './MappedInstances';
+import { getMetaValue } from '../../../../utils';
+
+const dagId = getMetaValue('dag_id');
const getTask = ({ taskId, runId, task }) => {
if (task.id === taskId) return task;
@@ -51,57 +54,62 @@ const getTask = ({ taskId, runId, task }) => {
};
const TaskInstance = ({ taskId, runId }) => {
- const { data: { groups = {} } } = useTreeData();
- const task = getTask({ taskId, runId, task: groups });
- if (!task) return null;
-
- const isGroup = !!task.children;
+ const { data: { groups = {}, dagRuns = [] } } = useTreeData();
+ const group = getTask({ taskId, runId, task: groups });
+ const run = dagRuns.find((r) => r.runId === runId);
+ const { executionDate } = run;
+ const { data: { tasks } } = useTasks(dagId);
+ if (!group) return null;
+ const task = tasks.find((t) => t.taskId === taskId);
+ const operator = task && task.classRef && task.classRef.className ? task.classRef.className : '';
- const instance = task.instances.find((ti) => ti.runId === runId);
+ const isGroup = !!group.children;
+ const { isMapped, extraLinks } = group;
- const {
- dagId,
- executionDate,
- tryNumber,
- } = instance;
+ const instance = group.instances.find((ti) => ti.runId === runId);
return (
<Box fontSize="12px" py="4px">
{!isGroup && (
- <TaskNav instance={instance} isMapped={task.isMapped} />
+ <TaskNav
+ taskId={taskId}
+ isMapped={isMapped}
+ executionDate={executionDate}
+ operator={operator}
+ />
)}
{!isGroup && (
<>
<VStack justifyContent="center" divider={<StackDivider my={3} />} my={3}>
- <RunAction runId={runId} taskId={task.id} dagId={dagId} />
+ <RunAction runId={runId} taskId={taskId} dagId={dagId} />
<ClearAction
runId={runId}
- taskId={task.id}
+ taskId={taskId}
dagId={dagId}
executionDate={executionDate}
/>
- <MarkFailedAction runId={runId} taskId={task.id} dagId={dagId} />
- <MarkSuccessAction runId={runId} taskId={task.id} dagId={dagId} />
+ <MarkFailedAction runId={runId} taskId={taskId} dagId={dagId} />
+ <MarkSuccessAction runId={runId} taskId={taskId} dagId={dagId} />
</VStack>
<Divider my={2} />
</>
)}
- {!task.isMapped && (
+ {!isMapped && (
<Logs
dagId={dagId}
taskId={taskId}
executionDate={executionDate}
- tryNumber={tryNumber}
+ tryNumber={instance.tryNumber}
/>
)}
- <Details instance={instance} task={task} />
+ <Details instance={instance} group={group} operator={operator} />
<ExtraLinks
taskId={taskId}
dagId={dagId}
executionDate={executionDate}
- extraLinks={task.extraLinks}
+ extraLinks={extraLinks}
/>
- {task.isMapped && (
+ {isMapped && (
<MappedInstances dagId={dagId} runId={runId} taskId={taskId} />
)}
</Box>
diff --git a/airflow/www/static/js/tree/details/content/taskInstance/taskActions/MarkFailed.jsx b/airflow/www/static/js/tree/details/content/taskInstance/taskActions/MarkFailed.jsx
index 7b61de388e..fe277c9eef 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/taskActions/MarkFailed.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/taskActions/MarkFailed.jsx
@@ -72,7 +72,7 @@ const MarkFailed = ({
setAffectedTasks(data);
onOpen();
} catch (e) {
- console.log(e);
+ console.error(e);
}
};
diff --git a/airflow/www/static/js/tree/treeDataUtils.js b/airflow/www/static/js/tree/treeDataUtils.js
index 680f2e63ac..95171652b7 100644
--- a/airflow/www/static/js/tree/treeDataUtils.js
+++ b/airflow/www/static/js/tree/treeDataUtils.js
@@ -19,7 +19,7 @@
import camelcaseKeys from 'camelcase-keys';
-export const areActiveRuns = (runs) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0;
+export const areActiveRuns = (runs = []) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0;
export const formatData = (data, emptyData) => {
if (!data || !Object.keys(data).length) {
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index 40b88bcd4d..fc29d62027 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -114,15 +114,12 @@ def get_mapped_summary(parent_instance, task_instances):
else parent_instance.try_number
)
return {
- 'dag_id': parent_instance.dag_id,
'task_id': parent_instance.task_id,
'run_id': parent_instance.run_id,
'state': group_state,
'start_date': group_start_date,
'end_date': group_end_date,
'mapped_states': mapped_states,
- 'operator': parent_instance.operator,
- 'execution_date': datetime_to_string(parent_instance.execution_date),
'try_number': try_count,
}
@@ -143,15 +140,12 @@ def encode_ti(
)
return {
'task_id': task_instance.task_id,
- 'dag_id': task_instance.dag_id,
'run_id': task_instance.run_id,
'map_index': task_instance.map_index,
'state': task_instance.state,
'duration': task_instance.duration,
'start_date': datetime_to_string(task_instance.start_date),
'end_date': datetime_to_string(task_instance.end_date),
- 'operator': task_instance.operator,
- 'execution_date': datetime_to_string(task_instance.execution_date),
'try_number': try_count,
}
@@ -161,7 +155,6 @@ def encode_dag_run(dag_run: Optional[models.DagRun]) -> Optional[Dict[str, Any]]
return None
return {
- 'dag_id': dag_run.dag_id,
'run_id': dag_run.run_id,
'start_date': datetime_to_string(dag_run.start_date),
'end_date': datetime_to_string(dag_run.end_date),