You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2021/11/16 00:24:30 UTC
[airflow] branch main updated: update tree data fetching (#19605)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d3ccc91 update tree data fetching (#19605)
d3ccc91 is described below
commit d3ccc91ba4af069d3402d458a1f0ca01c3ffb863
Author: Brent Bovenzi <br...@gmail.com>
AuthorDate: Mon Nov 15 18:24:02 2021 -0600
update tree data fetching (#19605)
- add `base_date` to refresh api request
- sort runs only on the webserver
- add test for auto-refresh stop
---
airflow/www/jest-setup.js | 6 ++++-
airflow/www/package.json | 1 +
airflow/www/static/js/tree/useTreeData.js | 15 ++++++-----
airflow/www/static/js/tree/useTreeData.test.js | 36 +++++++++++++++++---------
airflow/www/templates/airflow/tree.html | 1 +
airflow/www/views.py | 3 ++-
airflow/www/yarn.lock | 22 +++++++++++++++-
7 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js
index cbab0b6..c8ff532 100644
--- a/airflow/www/jest-setup.js
+++ b/airflow/www/jest-setup.js
@@ -1,3 +1,5 @@
+// We need this lint rule for now because these are only dev-dependencies
+/* eslint-disable import/no-extraneous-dependencies */
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,10 @@
* under the License.
*/
-// eslint-disable-next-line import/no-extraneous-dependencies
import '@testing-library/jest-dom';
+import { enableFetchMocks } from 'jest-fetch-mock';
+
+enableFetchMocks();
// Mock a global object we use across the app
global.stateColors = {
diff --git a/airflow/www/package.json b/airflow/www/package.json
index a77e2eb..c221982 100644
--- a/airflow/www/package.json
+++ b/airflow/www/package.json
@@ -55,6 +55,7 @@
"file-loader": "^6.0.0",
"imports-loader": "^1.1.0",
"jest": "^27.3.1",
+ "jest-fetch-mock": "^3.0.3",
"mini-css-extract-plugin": "1.6.0",
"moment": "^2.29.1",
"moment-locales-webpack-plugin": "^1.2.0",
diff --git a/airflow/www/static/js/tree/useTreeData.js b/airflow/www/static/js/tree/useTreeData.js
index 7d7f8f0..4dcfaf2 100644
--- a/airflow/www/static/js/tree/useTreeData.js
+++ b/airflow/www/static/js/tree/useTreeData.js
@@ -31,6 +31,7 @@ const treeDataUrl = getMetaValue('tree_data');
const numRuns = getMetaValue('num_runs');
const urlRoot = getMetaValue('root');
const isPaused = getMetaValue('is_paused');
+const baseDate = getMetaValue('base_date');
const areActiveRuns = (runs) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0;
@@ -46,20 +47,19 @@ const formatData = (data) => {
if (typeof data === 'string') formattedData = JSON.parse(data);
// change from pacal to camelcase
formattedData = camelcaseKeys(formattedData, { deep: true });
- // make sure dagRuns are sorted by date
- formattedData.dagRuns = formattedData.dagRuns
- .sort((a, b) => new Date(a.dataIntervalStart) - new Date(b.dataIntervalStart));
return formattedData;
};
const useTreeData = () => {
const [data, setData] = useState(formatData(treeData));
const defaultIsOpen = isPaused !== 'True' && !JSON.parse(localStorage.getItem('disableAutoRefresh')) && areActiveRuns(data.dagRuns);
- const { isOpen: isRefreshOn, onToggle } = useDisclosure({ defaultIsOpen });
+ const { isOpen: isRefreshOn, onToggle, onClose } = useDisclosure({ defaultIsOpen });
const handleRefresh = useCallback(async () => {
try {
- const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}&root=${urlRoot}`);
+ const root = urlRoot ? `&root=${urlRoot}` : '';
+ const base = baseDate ? `&base_date=${baseDate}` : '';
+ const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}${root}${base}`);
let newData = await resp.json();
if (newData) {
newData = formatData(newData);
@@ -67,12 +67,13 @@ const useTreeData = () => {
setData(newData);
}
// turn off auto refresh if there are no active runs
- if (!areActiveRuns(newData.dagRuns)) onToggle();
+ if (!areActiveRuns(newData.dagRuns)) onClose();
}
} catch (e) {
+ onClose();
console.error(e);
}
- }, [data, onToggle]);
+ }, [data, onClose]);
const onToggleRefresh = () => {
if (isRefreshOn) {
diff --git a/airflow/www/static/js/tree/useTreeData.test.js b/airflow/www/static/js/tree/useTreeData.test.js
index 5c511ae..85d0e21 100644
--- a/airflow/www/static/js/tree/useTreeData.test.js
+++ b/airflow/www/static/js/tree/useTreeData.test.js
@@ -20,11 +20,11 @@
import { renderHook } from '@testing-library/react-hooks';
import useTreeData from './useTreeData';
-/* global describe, test, expect */
+/* global describe, test, expect, fetch, beforeEach */
global.autoRefreshInterval = 5;
-const treeData = {
+const pendingTreeData = {
groups: {},
dag_runs: [
{
@@ -41,9 +41,18 @@ const treeData = {
],
};
+const finalTreeData = {
+ groups: {},
+ dag_runs: [{ ...pendingTreeData.dag_runs[0], state: 'failed' }],
+};
+
describe('Test useTreeData hook', () => {
+ beforeEach(() => {
+ fetch.resetMocks();
+ });
+
test('data is valid camelcase json', () => {
- global.treeData = JSON.stringify(treeData);
+ global.treeData = JSON.stringify(pendingTreeData);
const { result } = renderHook(() => useTreeData());
const { data, isRefreshOn, onToggleRefresh } = result.current;
@@ -55,20 +64,23 @@ describe('Test useTreeData hook', () => {
expect(typeof onToggleRefresh).toBe('function');
});
- test('data with an unfinished state should have refresh on by default', () => {
- global.treeData = JSON.stringify(treeData);
+ test('queued run should have refreshOn by default and then turn off when run failed', async () => {
+ // return a dag run of failed during refresh
+ fetch.mockResponse(JSON.stringify(finalTreeData));
+ global.treeData = JSON.stringify(pendingTreeData);
+ global.autoRefreshInterval = 0.1;
- const { result } = renderHook(() => useTreeData());
- const { isRefreshOn } = result.current;
+ const { result, waitFor } = renderHook(() => useTreeData());
- expect(isRefreshOn).toBe(true);
+ expect(result.current.isRefreshOn).toBe(true);
+
+ await waitFor(() => expect(fetch).toBeCalled());
+
+ expect(result.current.isRefreshOn).toBe(false);
});
test('data with a finished state should have refresh off by default', () => {
- global.treeData = JSON.stringify({
- groups: {},
- dag_runs: [{ ...treeData.dag_runs[0], state: 'failed' }],
- });
+ global.treeData = JSON.stringify(finalTreeData);
const { result } = renderHook(() => useTreeData());
const { isRefreshOn } = result.current;
diff --git a/airflow/www/templates/airflow/tree.html b/airflow/www/templates/airflow/tree.html
index 728b8f4..d76de75 100644
--- a/airflow/www/templates/airflow/tree.html
+++ b/airflow/www/templates/airflow/tree.html
@@ -25,6 +25,7 @@
{{ super() }}
<meta name="num_runs" content="{{ num_runs }}">
<meta name="root" content="{{ root if root else '' }}">
+ <meta name="base_date" content="{{ request.args.get('base_date') if request.args.get('base_date') else '' }}">
{% endblock %}
{% block content %}
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 00db9df..d561cc7 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -288,7 +288,6 @@ def task_group_to_tree(task_item_or_group, dag, dag_runs, tis):
}
group_summaries = [get_summary(dr, children) for dr in dag_runs]
- group_summaries.reverse()
return {
'id': task_group.group_id,
@@ -2256,6 +2255,7 @@ class Airflow(AirflowBaseView):
.limit(num_runs)
.all()
)
+ dag_runs.reverse()
encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs]
dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs}
@@ -3001,6 +3001,7 @@ class Airflow(AirflowBaseView):
.limit(num_runs)
.all()
)
+ dag_runs.reverse()
encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs]
dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs}
min_date = min(dag_run_dates, default=None)
diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock
index e0f6be1..8434813 100644
--- a/airflow/www/yarn.lock
+++ b/airflow/www/yarn.lock
@@ -4064,6 +4064,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
safe-buffer "^5.0.1"
sha.js "^2.4.8"
+cross-fetch@^3.0.4:
+ version "3.1.4"
+ resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39"
+ integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==
+ dependencies:
+ node-fetch "2.6.1"
+
cross-spawn@^6.0.5:
version "6.0.5"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -6923,6 +6930,14 @@ jest-environment-node@^27.3.1:
jest-mock "^27.3.0"
jest-util "^27.3.1"
+jest-fetch-mock@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/jest-fetch-mock/-/jest-fetch-mock-3.0.3.tgz#31749c456ae27b8919d69824f1c2bd85fe0a1f3b"
+ integrity sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw==
+ dependencies:
+ cross-fetch "^3.0.4"
+ promise-polyfill "^8.1.3"
+
jest-get-type@^27.3.1:
version "27.3.1"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.3.1.tgz#a8a2b0a12b50169773099eee60a0e6dd11423eff"
@@ -8064,7 +8079,7 @@ node-fetch-h2@^2.3.0:
dependencies:
http2-client "^1.2.5"
-node-fetch@^2.6.1:
+node-fetch@2.6.1, node-fetch@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
@@ -9115,6 +9130,11 @@ promise-inflight@^1.0.1:
resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"
integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM=
+promise-polyfill@^8.1.3:
+ version "8.2.1"
+ resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-8.2.1.tgz#1fa955b325bee4f6b8a4311e18148d4e5b46d254"
+ integrity sha512-3p9zj0cEHbp7NVUxEYUWjQlffXqnXaZIMPkAO7HhFh8u5636xLRDHOUo2vpWSK0T2mqm6fKLXYn1KP6PAZ2gKg==
+
prompts@^2.0.1:
version "2.4.2"
resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"