You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2022/04/01 19:20:56 UTC
[airflow] 01/01: split treeData and /grid
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch clean-grid-data
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit bcc79c07ec72bedf1b9e7b28cbb8bf5c00726a1c
Author: Brent Bovenzi <br...@gmail.com>
AuthorDate: Fri Apr 1 15:17:01 2022 -0400
split treeData and /grid
---
airflow/www/static/js/tree/Tree.jsx | 2 +-
airflow/www/static/js/tree/api/useTreeData.js | 24 +++---
.../www/static/js/tree/api/useTreeData.test.jsx | 91 ----------------------
airflow/www/static/js/tree/context/autorefresh.jsx | 15 +---
airflow/www/static/js/tree/dagRuns/index.test.jsx | 80 +++++++++----------
airflow/www/static/js/tree/renderTaskRows.test.jsx | 25 ------
airflow/www/static/js/tree/treeDataUtils.js | 34 --------
airflow/www/templates/airflow/tree.html | 1 -
airflow/www/views.py | 15 ----
9 files changed, 53 insertions(+), 234 deletions(-)
diff --git a/airflow/www/static/js/tree/Tree.jsx b/airflow/www/static/js/tree/Tree.jsx
index 6108293..508a986 100644
--- a/airflow/www/static/js/tree/Tree.jsx
+++ b/airflow/www/static/js/tree/Tree.jsx
@@ -120,7 +120,7 @@ const Tree = () => {
</Thead>
{/* TODO: remove hardcoded values. 665px is roughly the total heade+footer height */}
<Tbody display="block" width="100%" maxHeight="calc(100vh - 665px)" minHeight="500px" ref={tableRef} pr="10px">
- {renderTaskRows({
+ {groups.children && renderTaskRows({
task: groups, dagRunIds, tableWidth,
})}
</Tbody>
diff --git a/airflow/www/static/js/tree/api/useTreeData.js b/airflow/www/static/js/tree/api/useTreeData.js
index 83d638f..ed83a6d 100644
--- a/airflow/www/static/js/tree/api/useTreeData.js
+++ b/airflow/www/static/js/tree/api/useTreeData.js
@@ -17,13 +17,13 @@
* under the License.
*/
-/* global treeData, autoRefreshInterval, fetch */
+/* global autoRefreshInterval */
import { useQuery } from 'react-query';
+import axios from 'axios';
import { getMetaValue } from '../../utils';
import { useAutoRefresh } from '../context/autorefresh';
-import { formatData, areActiveRuns } from '../treeDataUtils';
// dagId comes from dag.html
const dagId = getMetaValue('dag_id');
@@ -32,25 +32,23 @@ const numRuns = getMetaValue('num_runs');
const urlRoot = getMetaValue('root');
const baseDate = getMetaValue('base_date');
+const areActiveRuns = (runs) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0;
+
const useTreeData = () => {
const emptyData = {
dagRuns: [],
groups: {},
};
- const initialData = formatData(treeData, emptyData);
const { isRefreshOn, stopRefresh } = useAutoRefresh();
return useQuery('treeData', async () => {
try {
const root = urlRoot ? `&root=${urlRoot}` : '';
const base = baseDate ? `&base_date=${baseDate}` : '';
- const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}${root}${base}`);
- if (resp) {
- let newData = await resp.json();
- newData = formatData(newData);
- // turn off auto refresh if there are no active runs
- if (!areActiveRuns(newData.dagRuns)) stopRefresh();
- return newData;
- }
+ const data = await axios.get(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}${root}${base}`);
+ if (!data || !data.dagRuns) return emptyData;
+ // turn off auto refresh if there are no active runs
+ if (!areActiveRuns(data.dagRuns)) stopRefresh();
+ return data;
} catch (e) {
stopRefresh();
console.error(e);
@@ -61,9 +59,9 @@ const useTreeData = () => {
};
}, {
// only enabled and refetch if the refresh switch is on
- enabled: isRefreshOn,
+ // enabled: isRefreshOn,
refetchInterval: isRefreshOn && autoRefreshInterval * 1000,
- initialData,
+ initialData: emptyData,
});
};
diff --git a/airflow/www/static/js/tree/api/useTreeData.test.jsx b/airflow/www/static/js/tree/api/useTreeData.test.jsx
deleted file mode 100644
index ab6b73a..0000000
--- a/airflow/www/static/js/tree/api/useTreeData.test.jsx
+++ /dev/null
@@ -1,91 +0,0 @@
-/*!
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-import React from 'react';
-import { renderHook } from '@testing-library/react-hooks';
-import { QueryClient, QueryClientProvider } from 'react-query';
-import useTreeData from './useTreeData';
-import { AutoRefreshProvider } from '../context/autorefresh';
-
-/* global describe, test, expect, jest, beforeAll */
-
-const pendingTreeData = {
- groups: {},
- dag_runs: [
- {
- dag_id: 'example_python_operator',
- run_id: 'manual__2021-11-08T21:14:17.170046+00:00',
- start_date: null,
- end_date: null,
- state: 'queued',
- execution_date: '2021-11-08T21:14:17.170046+00:00',
- data_interval_start: '2021-11-08T21:14:17.170046+00:00',
- data_interval_end: '2021-11-08T21:14:17.170046+00:00',
- run_type: 'manual',
- },
- ],
-};
-
-const Wrapper = ({ children }) => {
- const queryClient = new QueryClient();
- return (
- <AutoRefreshProvider>
- <QueryClientProvider client={queryClient}>
- {children}
- </QueryClientProvider>
- </AutoRefreshProvider>
- );
-};
-
-describe('Test useTreeData hook', () => {
- beforeAll(() => {
- global.autoRefreshInterval = 5;
- global.fetch = jest.fn();
- });
-
- test('data is valid camelcase json', () => {
- global.treeData = JSON.stringify(pendingTreeData);
-
- const { result } = renderHook(() => useTreeData(), { wrapper: Wrapper });
- const { data } = result.current;
-
- expect(typeof data === 'object').toBe(true);
- expect(data.dagRuns).toBeDefined();
- expect(data.dag_runs).toBeUndefined();
- });
-
- test('Can handle no treeData', () => {
- global.treeData = null;
-
- const { result } = renderHook(() => useTreeData(), { wrapper: Wrapper });
- const { data } = result.current;
-
- expect(data.dagRuns).toStrictEqual([]);
- expect(data.groups).toStrictEqual({});
- });
-
- test('Can handle empty treeData object', () => {
- global.treeData = {};
-
- const { result } = renderHook(() => useTreeData(), { wrapper: Wrapper });
- const { data } = result.current;
-
- expect(data.dagRuns).toStrictEqual([]);
- expect(data.groups).toStrictEqual({});
- });
-});
diff --git a/airflow/www/static/js/tree/context/autorefresh.jsx b/airflow/www/static/js/tree/context/autorefresh.jsx
index 77ca7f3..b889d9e 100644
--- a/airflow/www/static/js/tree/context/autorefresh.jsx
+++ b/airflow/www/static/js/tree/context/autorefresh.jsx
@@ -17,11 +17,10 @@
* under the License.
*/
-/* global localStorage, treeData, document */
+/* global localStorage, document */
import React, { useContext, useState, useEffect } from 'react';
import { getMetaValue } from '../../utils';
-import { formatData, areActiveRuns } from '../treeDataUtils';
const autoRefreshKey = 'disabledAutoRefresh';
@@ -31,17 +30,9 @@ const isRefreshDisabled = JSON.parse(localStorage.getItem(autoRefreshKey));
const AutoRefreshContext = React.createContext(null);
export const AutoRefreshProvider = ({ children }) => {
- let dagRuns = [];
- try {
- const data = JSON.parse(treeData);
- if (data.dag_runs) dagRuns = formatData(data.dag_runs);
- } catch {
- dagRuns = [];
- }
const [isPaused, setIsPaused] = useState(initialIsPaused);
- const isActive = areActiveRuns(dagRuns);
const isRefreshAllowed = !(isPaused || isRefreshDisabled);
- const initialState = isRefreshAllowed && isActive;
+ const initialState = isRefreshAllowed;
const [isRefreshOn, setRefresh] = useState(initialState);
@@ -67,7 +58,7 @@ export const AutoRefreshProvider = ({ children }) => {
setIsPaused(!e.value);
if (!e.value) {
stopRefresh();
- } else if (isActive) {
+ } else {
setRefresh(true);
}
};
diff --git a/airflow/www/static/js/tree/dagRuns/index.test.jsx b/airflow/www/static/js/tree/dagRuns/index.test.jsx
index 5faa8b6..a7499f4 100644
--- a/airflow/www/static/js/tree/dagRuns/index.test.jsx
+++ b/airflow/www/static/js/tree/dagRuns/index.test.jsx
@@ -17,7 +17,7 @@
* under the License.
*/
-/* global describe, test, expect */
+/* global describe, test, jest, expect */
import React from 'react';
import { render } from '@testing-library/react';
@@ -30,8 +30,12 @@ import { ContainerRefProvider } from '../context/containerRef';
import { SelectionProvider } from '../context/selection';
import { TimezoneProvider } from '../context/timezone';
import { AutoRefreshProvider } from '../context/autorefresh';
+import { useTreeData } from '../api';
+
+jest.mock('../api');
global.moment = moment;
+global.autoRefreshInterval = 0;
const Wrapper = ({ children }) => {
const queryClient = new QueryClient();
@@ -80,13 +84,37 @@ describe('Test DagRuns', () => {
runType: 'manual',
startDate: '2021-11-09T00:19:43.023200+00:00',
endDate: '2021-11-09T00:22:18.607167+00:00',
+ executionDate: '2021-11-08T21:14:19.704433+00:00',
+ },
+ {
+ dagId: 'dagId',
+ runId: 'run3',
+ dataIntervalStart: new Date(),
+ dataIntervalEnd: new Date(),
+ startDate: '2021-11-08T21:14:19.704433+00:00',
+ endDate: '2021-11-08T21:17:13.206426+00:00',
+ state: 'failed',
+ runType: 'scheduled',
+ executionDate: '2021-11-08T21:14:19.704433+00:00',
+ },
+ {
+ dagId: 'dagId',
+ runId: 'run4',
+ dataIntervalStart: new Date(),
+ dataIntervalEnd: new Date(),
+ state: 'success',
+ runType: 'manual',
+ startDate: '2021-11-09T00:19:43.023200+00:00',
+ endDate: '2021-11-09T00:22:18.607167+00:00',
+ executionDate: '2021-11-08T21:14:19.704433+00:00',
},
];
test('Durations and manual run arrow render correctly, but without any date ticks', () => {
- global.treeData = JSON.stringify({
- groups: {},
- dagRuns,
+ useTreeData.mockReturnValue({
+ data: {
+ dagRuns: dagRuns.slice(0, 2),
+ },
});
const { queryAllByTestId, getByText, queryByText } = render(
<DagRuns />, { wrapper: Wrapper },
@@ -100,51 +128,19 @@ describe('Test DagRuns', () => {
});
test('Top date ticks appear when there are 4 or more runs', () => {
- global.treeData = JSON.stringify({
- groups: {},
- dagRuns: [
- ...dagRuns,
- {
- dagId: 'dagId',
- runId: 'run3',
- dataIntervalStart: new Date(),
- dataIntervalEnd: new Date(),
- startDate: '2021-11-08T21:14:19.704433+00:00',
- endDate: '2021-11-08T21:17:13.206426+00:00',
- state: 'failed',
- runType: 'scheduled',
- },
- {
- dagId: 'dagId',
- runId: 'run4',
- dataIntervalStart: new Date(),
- dataIntervalEnd: new Date(),
- state: 'success',
- runType: 'manual',
- startDate: '2021-11-09T00:19:43.023200+00:00',
- endDate: '2021-11-09T00:22:18.607167+00:00',
- },
- ],
+ useTreeData.mockReturnValue({
+ data: {
+ dagRuns,
+ },
});
const { getByText } = render(
<DagRuns />, { wrapper: Wrapper },
);
- expect(getByText(moment.utc(dagRuns[0].executionDate).format('MMM DD, HH:mm'))).toBeInTheDocument();
- });
-
- test('Handles empty data correctly', () => {
- global.treeData = {
- groups: {},
- dagRuns: [],
- };
- const { queryByTestId } = render(
- <DagRuns />, { wrapper: Wrapper },
- );
- expect(queryByTestId('run')).toBeNull();
+ expect(getByText(moment.utc('2021-11-08T21:14:19.704433+00:00').format('MMM DD, HH:mm'))).toBeInTheDocument();
});
test('Handles no data correctly', () => {
- global.treeData = {};
+ useTreeData.mockReturnValue({ data: {} });
const { queryByTestId } = render(
<DagRuns />, { wrapper: Wrapper },
);
diff --git a/airflow/www/static/js/tree/renderTaskRows.test.jsx b/airflow/www/static/js/tree/renderTaskRows.test.jsx
index b163d62..444462b 100644
--- a/airflow/www/static/js/tree/renderTaskRows.test.jsx
+++ b/airflow/www/static/js/tree/renderTaskRows.test.jsx
@@ -117,7 +117,6 @@ const Wrapper = ({ children }) => {
describe('Test renderTaskRows', () => {
test('Group defaults to closed but clicking on the name will open a group', () => {
- global.treeData = mockTreeData;
const dagRunIds = mockTreeData.dagRuns.map((dr) => dr.runId);
const task = mockTreeData.groups;
@@ -140,30 +139,6 @@ describe('Test renderTaskRows', () => {
});
test('Still renders names if there are no instances', () => {
- global.treeData = {
- groups: {
- id: null,
- label: null,
- children: [
- {
- extraLinks: [],
- id: 'group_1',
- label: 'group_1',
- instances: [],
- children: [
- {
- id: 'group_1.task_1',
- label: 'group_1.task_1',
- extraLinks: [],
- instances: [],
- },
- ],
- },
- ],
- instances: [],
- },
- dagRuns: [],
- };
const task = mockTreeData.groups;
const { queryByTestId, getByText } = render(
diff --git a/airflow/www/static/js/tree/treeDataUtils.js b/airflow/www/static/js/tree/treeDataUtils.js
deleted file mode 100644
index 680f2e6..0000000
--- a/airflow/www/static/js/tree/treeDataUtils.js
+++ /dev/null
@@ -1,34 +0,0 @@
-/*!
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import camelcaseKeys from 'camelcase-keys';
-
-export const areActiveRuns = (runs) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0;
-
-export const formatData = (data, emptyData) => {
- if (!data || !Object.keys(data).length) {
- return emptyData;
- }
- let formattedData = data;
- // Convert to json if needed
- if (typeof data === 'string') formattedData = JSON.parse(data);
- // change from pascal to camelcase
- formattedData = camelcaseKeys(formattedData, { deep: true });
- return formattedData;
-};
diff --git a/airflow/www/templates/airflow/tree.html b/airflow/www/templates/airflow/tree.html
index 1394cbc..eff656a 100644
--- a/airflow/www/templates/airflow/tree.html
+++ b/airflow/www/templates/airflow/tree.html
@@ -78,7 +78,6 @@
{% block tail_js %}
{{ super() }}
<script>
- const treeData = {{ data|tojson }};
const stateColors = {{ state_color_mapping|tojson }};
const autoRefreshInterval = {{ auto_refresh_interval }};
</script>
diff --git a/airflow/www/views.py b/airflow/www/views.py
index f904636..3ca7801 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2530,7 +2530,6 @@ class Airflow(AirflowBaseView):
.all()
)
dag_runs.reverse()
- encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs]
dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs}
max_date = max(dag_run_dates, default=None)
@@ -2550,18 +2549,6 @@ class Airflow(AirflowBaseView):
else:
external_log_name = None
- min_date = min(dag_run_dates, default=None)
-
- tis = dag.get_task_instances(start_date=min_date, end_date=base_date, session=session)
-
- data = {
- 'groups': task_group_to_tree(dag.task_group, dag, dag_runs, tis, session),
- 'dag_runs': encoded_runs,
- }
-
- # avoid spaces to reduce payload size
- data = htmlsafe_json_dumps(data, separators=(',', ':'))
-
return self.render_template(
'airflow/tree.html',
operators=sorted({op.task_type: op for op in dag.tasks}.values(), key=lambda x: x.task_type),
@@ -2569,7 +2556,6 @@ class Airflow(AirflowBaseView):
form=form,
dag=dag,
doc_md=doc_md,
- data=data,
num_runs=num_runs,
show_external_log_redirect=task_log_reader.supports_external_link,
external_log_name=external_log_name,
@@ -3429,7 +3415,6 @@ class Airflow(AirflowBaseView):
'dag_runs': encoded_runs,
}
- # avoid spaces to reduce payload size
return htmlsafe_json_dumps(data, separators=(',', ':'))
@expose('/robots.txt')