You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2021/11/15 22:28:11 UTC
[airflow] branch autorefresh-w-basedate created (now cbc86a1)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a change to branch autorefresh-w-basedate
in repository https://gitbox.apache.org/repos/asf/airflow.git.
at cbc86a1 update tree data fetching
This branch includes the following new commits:
new cbc86a1 update tree data fetching
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airflow] 01/01: update tree data fetching
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch autorefresh-w-basedate
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cbc86a1bd210ebefeb12c759bad3b3e391244f6d
Author: Brent Bovenzi <br...@gmail.com>
AuthorDate: Mon Nov 15 16:12:54 2021 -0600
update tree data fetching
- include `base_date` in refresh api request
- handle run order in webserver
- add test for refresh to auto-stop
---
airflow/www/jest-setup.js | 6 ++++-
airflow/www/package.json | 1 +
airflow/www/static/js/tree/useTreeData.js | 15 ++++++-----
airflow/www/static/js/tree/useTreeData.test.js | 36 +++++++++++++++++---------
airflow/www/templates/airflow/tree.html | 1 +
airflow/www/views.py | 3 ++-
airflow/www/yarn.lock | 22 +++++++++++++++-
7 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js
index cbab0b6..c8ff532 100644
--- a/airflow/www/jest-setup.js
+++ b/airflow/www/jest-setup.js
@@ -1,3 +1,5 @@
+// We need this lint rule for now because these are only dev-dependencies
+/* eslint-disable import/no-extraneous-dependencies */
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,10 @@
* under the License.
*/
-// eslint-disable-next-line import/no-extraneous-dependencies
import '@testing-library/jest-dom';
+import { enableFetchMocks } from 'jest-fetch-mock';
+
+enableFetchMocks();
// Mock a global object we use across the app
global.stateColors = {
diff --git a/airflow/www/package.json b/airflow/www/package.json
index a77e2eb..c221982 100644
--- a/airflow/www/package.json
+++ b/airflow/www/package.json
@@ -55,6 +55,7 @@
"file-loader": "^6.0.0",
"imports-loader": "^1.1.0",
"jest": "^27.3.1",
+ "jest-fetch-mock": "^3.0.3",
"mini-css-extract-plugin": "1.6.0",
"moment": "^2.29.1",
"moment-locales-webpack-plugin": "^1.2.0",
diff --git a/airflow/www/static/js/tree/useTreeData.js b/airflow/www/static/js/tree/useTreeData.js
index 7d7f8f0..4dcfaf2 100644
--- a/airflow/www/static/js/tree/useTreeData.js
+++ b/airflow/www/static/js/tree/useTreeData.js
@@ -31,6 +31,7 @@ const treeDataUrl = getMetaValue('tree_data');
const numRuns = getMetaValue('num_runs');
const urlRoot = getMetaValue('root');
const isPaused = getMetaValue('is_paused');
+const baseDate = getMetaValue('base_date');
const areActiveRuns = (runs) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0;
@@ -46,20 +47,19 @@ const formatData = (data) => {
if (typeof data === 'string') formattedData = JSON.parse(data);
// change from pacal to camelcase
formattedData = camelcaseKeys(formattedData, { deep: true });
- // make sure dagRuns are sorted by date
- formattedData.dagRuns = formattedData.dagRuns
- .sort((a, b) => new Date(a.dataIntervalStart) - new Date(b.dataIntervalStart));
return formattedData;
};
const useTreeData = () => {
const [data, setData] = useState(formatData(treeData));
const defaultIsOpen = isPaused !== 'True' && !JSON.parse(localStorage.getItem('disableAutoRefresh')) && areActiveRuns(data.dagRuns);
- const { isOpen: isRefreshOn, onToggle } = useDisclosure({ defaultIsOpen });
+ const { isOpen: isRefreshOn, onToggle, onClose } = useDisclosure({ defaultIsOpen });
const handleRefresh = useCallback(async () => {
try {
- const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}&root=${urlRoot}`);
+ const root = urlRoot ? `&root=${urlRoot}` : '';
+ const base = baseDate ? `&base_date=${baseDate}` : '';
+ const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}${root}${base}`);
let newData = await resp.json();
if (newData) {
newData = formatData(newData);
@@ -67,12 +67,13 @@ const useTreeData = () => {
setData(newData);
}
// turn off auto refresh if there are no active runs
- if (!areActiveRuns(newData.dagRuns)) onToggle();
+ if (!areActiveRuns(newData.dagRuns)) onClose();
}
} catch (e) {
+ onClose();
console.error(e);
}
- }, [data, onToggle]);
+ }, [data, onClose]);
const onToggleRefresh = () => {
if (isRefreshOn) {
diff --git a/airflow/www/static/js/tree/useTreeData.test.js b/airflow/www/static/js/tree/useTreeData.test.js
index 5c511ae..85d0e21 100644
--- a/airflow/www/static/js/tree/useTreeData.test.js
+++ b/airflow/www/static/js/tree/useTreeData.test.js
@@ -20,11 +20,11 @@
import { renderHook } from '@testing-library/react-hooks';
import useTreeData from './useTreeData';
-/* global describe, test, expect */
+/* global describe, test, expect, fetch, beforeEach */
global.autoRefreshInterval = 5;
-const treeData = {
+const pendingTreeData = {
groups: {},
dag_runs: [
{
@@ -41,9 +41,18 @@ const treeData = {
],
};
+const finalTreeData = {
+ groups: {},
+ dag_runs: [{ ...pendingTreeData.dag_runs[0], state: 'failed' }],
+};
+
describe('Test useTreeData hook', () => {
+ beforeEach(() => {
+ fetch.resetMocks();
+ });
+
test('data is valid camelcase json', () => {
- global.treeData = JSON.stringify(treeData);
+ global.treeData = JSON.stringify(pendingTreeData);
const { result } = renderHook(() => useTreeData());
const { data, isRefreshOn, onToggleRefresh } = result.current;
@@ -55,20 +64,23 @@ describe('Test useTreeData hook', () => {
expect(typeof onToggleRefresh).toBe('function');
});
- test('data with an unfinished state should have refresh on by default', () => {
- global.treeData = JSON.stringify(treeData);
+ test('queued run should have refreshOn by default and then turn off when run failed', async () => {
+ // return a dag run of failed during refresh
+ fetch.mockResponse(JSON.stringify(finalTreeData));
+ global.treeData = JSON.stringify(pendingTreeData);
+ global.autoRefreshInterval = 0.1;
- const { result } = renderHook(() => useTreeData());
- const { isRefreshOn } = result.current;
+ const { result, waitFor } = renderHook(() => useTreeData());
- expect(isRefreshOn).toBe(true);
+ expect(result.current.isRefreshOn).toBe(true);
+
+ await waitFor(() => expect(fetch).toBeCalled());
+
+ expect(result.current.isRefreshOn).toBe(false);
});
test('data with a finished state should have refresh off by default', () => {
- global.treeData = JSON.stringify({
- groups: {},
- dag_runs: [{ ...treeData.dag_runs[0], state: 'failed' }],
- });
+ global.treeData = JSON.stringify(finalTreeData);
const { result } = renderHook(() => useTreeData());
const { isRefreshOn } = result.current;
diff --git a/airflow/www/templates/airflow/tree.html b/airflow/www/templates/airflow/tree.html
index 728b8f4..d76de75 100644
--- a/airflow/www/templates/airflow/tree.html
+++ b/airflow/www/templates/airflow/tree.html
@@ -25,6 +25,7 @@
{{ super() }}
<meta name="num_runs" content="{{ num_runs }}">
<meta name="root" content="{{ root if root else '' }}">
+ <meta name="base_date" content="{{ request.args.get('base_date') if request.args.get('base_date') else '' }}">
{% endblock %}
{% block content %}
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 5fe5966..c958944 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -288,7 +288,6 @@ def task_group_to_tree(task_item_or_group, dag, dag_runs, tis):
}
group_summaries = [get_summary(dr, children) for dr in dag_runs]
- group_summaries.reverse()
return {
'id': task_group.group_id,
@@ -2272,6 +2271,7 @@ class Airflow(AirflowBaseView):
.limit(num_runs)
.all()
)
+ dag_runs.reverse()
encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs]
dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs}
@@ -3017,6 +3017,7 @@ class Airflow(AirflowBaseView):
.limit(num_runs)
.all()
)
+ dag_runs.reverse()
encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs]
dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs}
min_date = min(dag_run_dates, default=None)
diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock
index e0f6be1..8434813 100644
--- a/airflow/www/yarn.lock
+++ b/airflow/www/yarn.lock
@@ -4064,6 +4064,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
safe-buffer "^5.0.1"
sha.js "^2.4.8"
+cross-fetch@^3.0.4:
+ version "3.1.4"
+ resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39"
+ integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==
+ dependencies:
+ node-fetch "2.6.1"
+
cross-spawn@^6.0.5:
version "6.0.5"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -6923,6 +6930,14 @@ jest-environment-node@^27.3.1:
jest-mock "^27.3.0"
jest-util "^27.3.1"
+jest-fetch-mock@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/jest-fetch-mock/-/jest-fetch-mock-3.0.3.tgz#31749c456ae27b8919d69824f1c2bd85fe0a1f3b"
+ integrity sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw==
+ dependencies:
+ cross-fetch "^3.0.4"
+ promise-polyfill "^8.1.3"
+
jest-get-type@^27.3.1:
version "27.3.1"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.3.1.tgz#a8a2b0a12b50169773099eee60a0e6dd11423eff"
@@ -8064,7 +8079,7 @@ node-fetch-h2@^2.3.0:
dependencies:
http2-client "^1.2.5"
-node-fetch@^2.6.1:
+node-fetch@2.6.1, node-fetch@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
@@ -9115,6 +9130,11 @@ promise-inflight@^1.0.1:
resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"
integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM=
+promise-polyfill@^8.1.3:
+ version "8.2.1"
+ resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-8.2.1.tgz#1fa955b325bee4f6b8a4311e18148d4e5b46d254"
+ integrity sha512-3p9zj0cEHbp7NVUxEYUWjQlffXqnXaZIMPkAO7HhFh8u5636xLRDHOUo2vpWSK0T2mqm6fKLXYn1KP6PAZ2gKg==
+
prompts@^2.0.1:
version "2.4.2"
resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"