You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2022/06/28 22:57:08 UTC
[airflow] branch main updated: Grid task log, multi select and ts files migration. (#24623)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 75db755f4b Grid task log, multi select and ts files migration. (#24623)
75db755f4b is described below
commit 75db755f4b06b4cfdd3eb2651dbf88ddba2d831f
Author: pierrejeambrun <pi...@gmail.com>
AuthorDate: Wed Jun 29 00:56:44 2022 +0200
Grid task log, multi select and ts files migration. (#24623)
* Multi select + ts files migration.
* Fix conflicts and transfert tests to ts files
* Add multi select on file source.
* Update placeholder color
* Update scrollIntoview.
---
airflow/www/package.json | 1 +
.../js/grid/api/{useTaskLog.js => useTaskLog.tsx} | 18 +-
.../useTaskLog.js => components/MultiSelect.tsx} | 52 ++--
.../Logs/{LogLink.test.jsx => LogLink.test.tsx} | 4 +-
.../taskInstance/Logs/{LogLink.jsx => LogLink.tsx} | 13 +-
.../details/content/taskInstance/Logs/index.jsx | 244 -----------------
.../Logs/{index.test.jsx => index.test.tsx} | 13 +-
.../details/content/taskInstance/Logs/index.tsx | 290 +++++++++++++++++++++
.../Logs/{utils.test.js => utils.test.tsx} | 47 ++--
.../taskInstance/Logs/{utils.js => utils.ts} | 43 ++-
.../js/grid/details/content/taskInstance/index.tsx | 4 +-
airflow/www/static/js/grid/types/index.ts | 10 +
airflow/www/yarn.lock | 70 ++++-
13 files changed, 485 insertions(+), 324 deletions(-)
diff --git a/airflow/www/package.json b/airflow/www/package.json
index ad11cc3a0d..45c9efb66b 100644
--- a/airflow/www/package.json
+++ b/airflow/www/package.json
@@ -82,6 +82,7 @@
"axios": "^0.26.0",
"bootstrap-3-typeahead": "^4.0.2",
"camelcase-keys": "^7.0.0",
+ "chakra-react-select": "^4.0.0",
"codemirror": "^5.59.1",
"d3": "^3.4.4",
"d3-shape": "^2.1.0",
diff --git a/airflow/www/static/js/grid/api/useTaskLog.js b/airflow/www/static/js/grid/api/useTaskLog.tsx
similarity index 69%
copy from airflow/www/static/js/grid/api/useTaskLog.js
copy to airflow/www/static/js/grid/api/useTaskLog.tsx
index 40041d1c33..8d56028629 100644
--- a/airflow/www/static/js/grid/api/useTaskLog.js
+++ b/airflow/www/static/js/grid/api/useTaskLog.tsx
@@ -17,23 +17,31 @@
* under the License.
*/
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
import { useQuery } from 'react-query';
import { getMetaValue } from '../../utils';
const taskLogApi = getMetaValue('task_log_api');
const useTaskLog = ({
- dagId, dagRunId, taskId, taskTryNumber, fullContent, enabled,
+ dagId, dagRunId, taskId, taskTryNumber, fullContent,
+}: {
+ dagId: string,
+ dagRunId: string,
+ taskId: string,
+ taskTryNumber: number,
+ fullContent: boolean,
}) => {
- const url = taskLogApi.replace('_DAG_RUN_ID_', dagRunId).replace('_TASK_ID_', taskId).replace(/-1$/, taskTryNumber);
+ let url: string = '';
+ if (taskLogApi) {
+ url = taskLogApi.replace('_DAG_RUN_ID_', dagRunId).replace('_TASK_ID_', taskId).replace(/-1$/, taskTryNumber.toString());
+ }
return useQuery(
['taskLogs', dagId, dagRunId, taskId, taskTryNumber, fullContent],
- () => axios.get(url, { headers: { Accept: 'text/plain' }, params: { full_content: fullContent } }),
+ () => axios.get<AxiosResponse, string>(url, { headers: { Accept: 'text/plain' }, params: { full_content: fullContent } }),
{
placeholderData: '',
- enabled,
},
);
};
diff --git a/airflow/www/static/js/grid/api/useTaskLog.js b/airflow/www/static/js/grid/components/MultiSelect.tsx
similarity index 51%
rename from airflow/www/static/js/grid/api/useTaskLog.js
rename to airflow/www/static/js/grid/components/MultiSelect.tsx
index 40041d1c33..f326c78e5d 100644
--- a/airflow/www/static/js/grid/api/useTaskLog.js
+++ b/airflow/www/static/js/grid/components/MultiSelect.tsx
@@ -17,25 +17,37 @@
* under the License.
*/
-import axios from 'axios';
-import { useQuery } from 'react-query';
-import { getMetaValue } from '../../utils';
+import React from 'react';
+import { Select } from 'chakra-react-select';
+import type { SelectComponent } from 'chakra-react-select';
-const taskLogApi = getMetaValue('task_log_api');
+const MultiSelect: SelectComponent = ({ chakraStyles, ...props }) => (
+ <Select
+ size="sm"
+ selectedOptionStyle="check"
+ {...props}
+ chakraStyles={{
+ dropdownIndicator: (provided) => ({
+ ...provided,
+ bg: 'transparent',
+ px: 2,
+ cursor: 'inherit',
+ }),
+ indicatorSeparator: (provided) => ({
+ ...provided,
+ display: 'none',
+ }),
+ menuList: (provided) => ({
+ ...provided,
+ py: 0,
+ }),
+ placeholder: (provided) => ({
+ ...provided,
+ color: 'inherit',
+ }),
+ ...chakraStyles,
+ }}
+ />
+);
-const useTaskLog = ({
- dagId, dagRunId, taskId, taskTryNumber, fullContent, enabled,
-}) => {
- const url = taskLogApi.replace('_DAG_RUN_ID_', dagRunId).replace('_TASK_ID_', taskId).replace(/-1$/, taskTryNumber);
-
- return useQuery(
- ['taskLogs', dagId, dagRunId, taskId, taskTryNumber, fullContent],
- () => axios.get(url, { headers: { Accept: 'text/plain' }, params: { full_content: fullContent } }),
- {
- placeholderData: '',
- enabled,
- },
- );
-};
-
-export default useTaskLog;
+export default MultiSelect;
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.test.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.test.tsx
similarity index 96%
rename from airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.test.jsx
rename to airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.test.tsx
index 9a47fe3b85..c6ae0ce49d 100644
--- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.test.jsx
+++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.test.tsx
@@ -40,7 +40,7 @@ describe('Test LogLink Component.', () => {
const linkElement = container.querySelector('a');
expect(linkElement).toBeDefined();
expect(linkElement).not.toHaveAttribute('target');
- expect(linkElement.href.includes(
+ expect(linkElement?.href.includes(
`?dag_id=dummyDagId&task_id=dummyTaskId&execution_date=2020%3A01%3A01T01%3A00%2B00%3A00&format=file&try_number=${tryNumber}`,
)).toBeTruthy();
});
@@ -60,7 +60,7 @@ describe('Test LogLink Component.', () => {
const linkElement = container.querySelector('a');
expect(linkElement).toBeDefined();
expect(linkElement).toHaveAttribute('target', '_blank');
- expect(linkElement.href.includes(
+ expect(linkElement?.href.includes(
`?dag_id=dummyDagId&task_id=dummyTaskId&execution_date=2020%3A01%3A01T01%3A00%2B00%3A00&try_number=${tryNumber}`,
)).toBeTruthy();
});
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.tsx
similarity index 85%
rename from airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.jsx
rename to airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.tsx
index 2e0f13f88d..4533f50513 100644
--- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.jsx
+++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/LogLink.tsx
@@ -21,20 +21,29 @@ import React from 'react';
import { getMetaValue } from '../../../../../utils';
import LinkButton from '../../../../components/LinkButton';
+import type { Dag, DagRun, TaskInstance } from '../../../../types';
const logsWithMetadataUrl = getMetaValue('logs_with_metadata_url');
const externalLogUrl = getMetaValue('external_log_url');
+interface Props {
+ dagId: Dag['id'];
+ taskId: TaskInstance['taskId'];
+ executionDate: DagRun['executionDate'];
+ isInternal?: boolean;
+ tryNumber: TaskInstance['tryNumber'];
+}
+
const LogLink = ({
dagId, taskId, executionDate, isInternal, tryNumber,
-}) => {
+}: Props) => {
let fullMetadataUrl = `${isInternal ? logsWithMetadataUrl : externalLogUrl
}?dag_id=${encodeURIComponent(dagId)
}&task_id=${encodeURIComponent(taskId)
}&execution_date=${encodeURIComponent(executionDate)
}`;
- if (isInternal) {
+ if (isInternal && tryNumber) {
fullMetadataUrl += `&format=file${tryNumber > 0 && `&try_number=${tryNumber}`}`;
} else {
fullMetadataUrl += `&try_number=${tryNumber}`;
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx
deleted file mode 100644
index 890d85feeb..0000000000
--- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx
+++ /dev/null
@@ -1,244 +0,0 @@
-/*!
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import React, {
- useRef, useState, useEffect, useMemo,
-} from 'react';
-import {
- Text,
- Box,
- Flex,
- Divider,
- Code,
- Button,
- Checkbox,
- Select,
-} from '@chakra-ui/react';
-
-import { getMetaValue } from '../../../../../utils';
-import LogLink from './LogLink';
-import useTaskLog from '../../../../api/useTaskLog';
-import LinkButton from '../../../../components/LinkButton';
-import { logLevel, parseLogs } from './utils';
-import { useTimezone } from '../../../../context/timezone';
-
-const showExternalLogRedirect = getMetaValue('show_external_log_redirect') === 'True';
-const externalLogName = getMetaValue('external_log_name');
-const logUrl = getMetaValue('log_url');
-
-const getLinkIndexes = (tryNumber) => {
- const internalIndexes = [];
- const externalIndexes = [];
-
- [...Array(tryNumber + 1 || 0)].forEach((_, index) => {
- if (index === 0 && tryNumber < 2) return;
- const isExternal = index !== 0 && showExternalLogRedirect;
- if (isExternal) {
- externalIndexes.push(index);
- } else {
- internalIndexes.push(index);
- }
- });
-
- return [internalIndexes, externalIndexes];
-};
-
-const Logs = ({
- dagId,
- dagRunId,
- taskId,
- executionDate,
- tryNumber,
-}) => {
- const [internalIndexes, externalIndexes] = getLinkIndexes(tryNumber);
- const [selectedAttempt, setSelectedAttempt] = useState(1);
- const [shouldRequestFullContent, setShouldRequestFullContent] = useState(false);
- const [wrap, setWrap] = useState(false);
- const [logLevelFilter, setLogLevelFilter] = useState('');
- const [fileSourceFilter, setFileSourceFilter] = useState('');
- const { timezone } = useTimezone();
- const { data, isSuccess } = useTaskLog({
- dagId,
- dagRunId,
- taskId,
- taskTryNumber: selectedAttempt,
- fullContent: shouldRequestFullContent,
- });
-
- const codeBlockBottomDiv = useRef(null);
-
- useEffect(() => {
- if (codeBlockBottomDiv.current) {
- codeBlockBottomDiv.current.scrollIntoView({ block: 'nearest', inline: 'nearest' });
- }
- }, [wrap, data]);
-
- const params = new URLSearchParams({
- task_id: taskId,
- execution_date: executionDate,
- }).toString();
-
- const { parsedLogs, fileSources = [] } = useMemo(
- () => parseLogs(
- data,
- timezone,
- logLevelFilter,
- fileSourceFilter,
- ),
- [data, fileSourceFilter, logLevelFilter, timezone],
- );
-
- useEffect(() => {
- // Reset fileSourceFilter and selected attempt when changing to
- // a task that do not have those filters anymore.
- if (!internalIndexes.includes(selectedAttempt)) {
- setSelectedAttempt(internalIndexes[0]);
- }
- if (fileSourceFilter && !fileSources.includes(fileSourceFilter)) {
- setFileSourceFilter('');
- }
- }, [data, internalIndexes, fileSourceFilter, fileSources, selectedAttempt]);
-
- return (
- <>
- {tryNumber > 0 && (
- <>
- <Text as="span"> (by attempts)</Text>
- <Flex my={1} justifyContent="space-between">
- <Flex flexWrap="wrap">
- {internalIndexes.map((index) => (
- <Button
- key={index}
- variant="ghost"
- colorScheme="blue"
- onClick={() => setSelectedAttempt(index)}
- data-testid={`log-attempt-select-button-${index}`}
- >
- {index}
- </Button>
- ))}
- </Flex>
- <Flex alignItems="center">
- <Box w="90px" mr={2}>
- <Select
- size="sm"
- value={logLevelFilter}
- onChange={(e) => setLogLevelFilter(e.target.value)}
- >
- <option value="" key="all">All Levels</option>
- {Object.values(logLevel).map((value) => (
- <option value={value} key={value}>{value}</option>
- ))}
- </Select>
- </Box>
- <Box w="110px">
- <Select
- size="sm"
- value={fileSourceFilter}
- onChange={(e) => setFileSourceFilter(e.target.value)}
- >
- <option value="" key="all">All File Sources</option>
- {fileSources.map((value) => (
- <option value={value} key={value}>{value}</option>
- ))}
- </Select>
- </Box>
- </Flex>
- <Flex alignItems="center">
- <Checkbox
- onChange={() => setWrap((previousState) => !previousState)}
- px={4}
- >
- <Text as="strong">Wrap</Text>
- </Checkbox>
- <Checkbox
- onChange={() => setShouldRequestFullContent((previousState) => !previousState)}
- px={4}
- data-testid="full-content-checkbox"
- >
- <Text as="strong">Full Logs</Text>
- </Checkbox>
- <LogLink
- index={selectedAttempt}
- dagId={dagId}
- taskId={taskId}
- executionDate={executionDate}
- isInternal
- />
- <LinkButton
- href={`${logUrl}&${params}`}
- >
- See More
- </LinkButton>
- </Flex>
- </Flex>
- {
- isSuccess && (
- <Code
- height={350}
- overflowY="scroll"
- p={3}
- pb={0}
- display="block"
- whiteSpace={wrap ? 'pre-wrap' : 'pre'}
- border="1px solid"
- borderRadius={3}
- borderColor="blue.500"
- >
- {parsedLogs}
- <div ref={codeBlockBottomDiv} />
- </Code>
- )
- }
- </>
- )}
- {externalLogName && externalIndexes.length > 0 && (
- <>
- <Box>
- <Text>
- View Logs in
- {' '}
- {externalLogName}
- {' '}
- (by attempts):
- </Text>
- <Flex flexWrap="wrap">
- {
- externalIndexes.map(
- (index) => (
- <LogLink
- key={index}
- dagId={dagId}
- taskId={taskId}
- executionDate={executionDate}
- tryNumber={index}
- />
- ),
- )
- }
- </Flex>
- </Box>
- <Divider my={2} />
- </>
- )}
- </>
- );
-};
-
-export default Logs;
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.tsx
similarity index 96%
rename from airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx
rename to airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.tsx
index a487081277..e0a510ace4 100644
--- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx
+++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.tsx
@@ -21,6 +21,7 @@
import React from 'react';
import { render, fireEvent } from '@testing-library/react';
+import type { UseQueryResult } from 'react-query';
import Logs from './index';
import * as useTaskLogModule from '../../../../api/useTaskLog';
@@ -45,14 +46,16 @@ AIRFLOW_CTX_DAG_OWNER=***
AIRFLOW_CTX_DAG_ID=test_ui_grid
`;
-let useTaskLogMock;
+let useTaskLogMock: jest.SpyInstance;
describe('Test Logs Component.', () => {
+ const returnValue = {
+ data: mockTaskLog,
+ isSuccess: true,
+ } as UseQueryResult<string, unknown>;
+
beforeEach(() => {
- useTaskLogMock = jest.spyOn(useTaskLogModule, 'default').mockImplementation(() => ({
- data: mockTaskLog,
- isSuccess: true,
- }));
+ useTaskLogMock = jest.spyOn(useTaskLogModule, 'default').mockImplementation(() => returnValue);
window.HTMLElement.prototype.scrollIntoView = jest.fn();
});
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.tsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.tsx
new file mode 100644
index 0000000000..ea68ab7407
--- /dev/null
+++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.tsx
@@ -0,0 +1,290 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, {
+ useRef, useState, useEffect, useMemo,
+} from 'react';
+import {
+ Text,
+ Box,
+ Flex,
+ Divider,
+ Code,
+ Button,
+ Checkbox,
+} from '@chakra-ui/react';
+
+import { getMetaValue } from '../../../../../utils';
+import LogLink from './LogLink';
+import useTaskLog from '../../../../api/useTaskLog';
+import LinkButton from '../../../../components/LinkButton';
+import { LogLevel, logLevelColorMapping, parseLogs } from './utils';
+import { useTimezone } from '../../../../context/timezone';
+import type { Dag, DagRun, TaskInstance } from '../../../../types';
+import MultiSelect from '../../../../components/MultiSelect';
+
+interface LogLevelOption {
+ label: LogLevel;
+ value: LogLevel;
+ color: string;
+}
+
+interface FileSourceOption {
+ label: string;
+ value: string;
+}
+
+const showExternalLogRedirect = getMetaValue('show_external_log_redirect') === 'True';
+const externalLogName = getMetaValue('external_log_name');
+const logUrl = getMetaValue('log_url');
+
+const getLinkIndexes = (tryNumber: number | undefined): Array<Array<number>> => {
+ const internalIndexes: Array<number> = [];
+ const externalIndexes: Array<number> = [];
+
+ if (tryNumber) {
+ [...Array(tryNumber + 1 || 0)].forEach((_, index) => {
+ if (index === 0 && tryNumber < 2) return;
+ const isExternal = index !== 0 && showExternalLogRedirect;
+ if (isExternal) {
+ externalIndexes.push(index);
+ } else {
+ internalIndexes.push(index);
+ }
+ });
+ }
+
+ return [internalIndexes, externalIndexes];
+};
+
+const logLevelOptions: Array<LogLevelOption> = Object.values(LogLevel).map(
+ (value): LogLevelOption => ({
+ label: value, value, color: logLevelColorMapping[value],
+ }),
+);
+
+interface Props {
+ dagId: Dag['id'];
+ dagRunId: DagRun['runId'];
+ taskId: TaskInstance['taskId'];
+ executionDate: DagRun['executionDate'];
+ tryNumber: TaskInstance['tryNumber'];
+}
+
+const Logs = ({
+ dagId,
+ dagRunId,
+ taskId,
+ executionDate,
+ tryNumber,
+}: Props) => {
+ const [internalIndexes, externalIndexes] = getLinkIndexes(tryNumber);
+ const [selectedAttempt, setSelectedAttempt] = useState(1);
+ const [shouldRequestFullContent, setShouldRequestFullContent] = useState(false);
+ const [wrap, setWrap] = useState(false);
+ const [logLevelFilters, setLogLevelFilters] = useState<Array<LogLevelOption>>([]);
+ const [fileSourceFilters, setFileSourceFilters] = useState<Array<FileSourceOption>>([]);
+ const { timezone } = useTimezone();
+ const { data, isSuccess } = useTaskLog({
+ dagId,
+ dagRunId,
+ taskId,
+ taskTryNumber: selectedAttempt,
+ fullContent: shouldRequestFullContent,
+ });
+
+ const params = new URLSearchParams({
+ task_id: taskId,
+ execution_date: executionDate,
+ }).toString();
+
+ const { parsedLogs, fileSources = [] } = useMemo(
+ () => parseLogs(
+ data,
+ timezone,
+ logLevelFilters.map((option) => option.value),
+ fileSourceFilters.map((option) => option.value),
+ ),
+ [data, fileSourceFilters, logLevelFilters, timezone],
+ );
+
+ const codeBlockBottomDiv = useRef<HTMLDivElement>(null);
+
+ useEffect(() => {
+ if (codeBlockBottomDiv.current) {
+ codeBlockBottomDiv.current.scrollIntoView({ block: 'nearest', inline: 'nearest' });
+ }
+ }, [wrap, parsedLogs]);
+
+ useEffect(() => {
+ // Reset fileSourceFilters and selected attempt when changing to
+ // a task that do not have those filters anymore.
+ if (!internalIndexes.includes(selectedAttempt)) {
+ setSelectedAttempt(internalIndexes[0]);
+ }
+
+ if (data && fileSourceFilters.length > 0
+ && fileSourceFilters.reduce(
+ (isSourceMissing, option) => (isSourceMissing || !fileSources.includes(option.value)),
+ false,
+ )) {
+ setFileSourceFilters([]);
+ }
+ }, [data, internalIndexes, fileSourceFilters, fileSources, selectedAttempt]);
+
+ return (
+ <>
+ {tryNumber! > 0 && (
+ <>
+ <Text as="span"> (by attempts)</Text>
+ <Flex my={1} justifyContent="space-between">
+ <Flex flexWrap="wrap">
+ {internalIndexes.map((index) => (
+ <Button
+ key={index}
+ variant="ghost"
+ colorScheme="blue"
+ onClick={() => setSelectedAttempt(index)}
+ data-testid={`log-attempt-select-button-${index}`}
+ >
+ {index}
+ </Button>
+ ))}
+ </Flex>
+ </Flex>
+ <Flex my={1} justifyContent="space-between">
+ <Flex alignItems="center" flexGrow={1} mr={10}>
+ <Box width="100%" mr={2}>
+ <MultiSelect
+ size="sm"
+ isMulti
+ options={logLevelOptions}
+ placeholder="All Levels"
+ value={logLevelFilters}
+ onChange={(options) => setLogLevelFilters([...options])}
+ chakraStyles={{
+ multiValue: (provided, ...rest) => ({
+ ...provided,
+ backgroundColor: rest[0].data.color,
+ }),
+ option: (provided, ...rest) => ({
+ ...provided,
+ borderLeft: 'solid 4px black',
+ borderColor: rest[0].data.color,
+ mt: 2,
+ }),
+ }}
+ />
+ </Box>
+ <Box width="100%">
+ <MultiSelect
+ size="sm"
+ isMulti
+ options={fileSources.map((fileSource) => ({
+ label: fileSource,
+ value: fileSource,
+ }))}
+ placeholder="All File Sources"
+ value={fileSourceFilters}
+ onChange={(options) => setFileSourceFilters([...options])}
+ />
+ </Box>
+ </Flex>
+ <Flex alignItems="center">
+ <Checkbox
+ onChange={() => setWrap((previousState) => !previousState)}
+ px={4}
+ >
+ <Text as="strong">Wrap</Text>
+ </Checkbox>
+ <Checkbox
+ onChange={() => setShouldRequestFullContent((previousState) => !previousState)}
+ px={4}
+ data-testid="full-content-checkbox"
+ >
+ <Text as="strong" whiteSpace="nowrap">Full Logs</Text>
+ </Checkbox>
+ <LogLink
+ dagId={dagId}
+ taskId={taskId}
+ executionDate={executionDate}
+ isInternal
+ tryNumber={tryNumber}
+ />
+ <LinkButton
+ href={`${logUrl}&${params}`}
+ >
+ See More
+ </LinkButton>
+ </Flex>
+ </Flex>
+ {
+ isSuccess && (
+ <Code
+ height={350}
+ overflowY="scroll"
+ p={3}
+ pb={0}
+ display="block"
+ whiteSpace={wrap ? 'pre-wrap' : 'pre'}
+ border="1px solid"
+ borderRadius={3}
+ borderColor="blue.500"
+ >
+ {parsedLogs}
+ <div ref={codeBlockBottomDiv} />
+ </Code>
+ )
+ }
+ </>
+ )}
+ {externalLogName && externalIndexes.length > 0 && (
+ <>
+ <Box>
+ <Text>
+ View Logs in
+ {' '}
+ {externalLogName}
+ {' '}
+ (by attempts):
+ </Text>
+ <Flex flexWrap="wrap">
+ {
+ externalIndexes.map(
+ (index) => (
+ <LogLink
+ key={index}
+ dagId={dagId}
+ taskId={taskId}
+ executionDate={executionDate}
+ tryNumber={index}
+ />
+ ),
+ )
+ }
+ </Flex>
+ </Box>
+ <Divider my={2} />
+ </>
+ )}
+ </>
+ );
+};
+
+export default Logs;
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.tsx
similarity index 83%
rename from airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js
rename to airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.tsx
index c4fe1f39ad..d47e5992eb 100644
--- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js
+++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.tsx
@@ -19,7 +19,7 @@
/* global describe, test, expect */
-import { parseLogs } from './utils';
+import { LogLevel, parseLogs } from './utils';
const mockTaskLog = `
5d28cfda3219
@@ -47,8 +47,8 @@ describe('Test Logs Utils.', () => {
const { parsedLogs, fileSources } = parseLogs(
mockTaskLog,
'UTC',
- null,
- null,
+ [],
+ [],
);
expect(parsedLogs).toContain('2022-06-04, 00:00:01 UTC');
@@ -61,32 +61,37 @@ describe('Test Logs Utils.', () => {
const result = parseLogs(
mockTaskLog,
'America/Los_Angeles',
- null,
- null,
+ [],
+ [],
);
expect(result.parsedLogs).toContain('2022-06-03, 17:00:01 PDT');
});
test.each([
- { logLevelFilter: 'INFO', expectedNumberOfLines: 11, expectedNumberOfFileSources: 4 },
- { logLevelFilter: 'WARNING', expectedNumberOfLines: 1, expectedNumberOfFileSources: 1 },
+ { logLevelFilters: [LogLevel.INFO], expectedNumberOfLines: 11, expectedNumberOfFileSources: 4 },
+ {
+ logLevelFilters: [LogLevel.WARNING],
+ expectedNumberOfLines: 1,
+ expectedNumberOfFileSources: 1,
+ },
])(
- 'Filtering logs on $logLevelFilter level should return $expectedNumberOfLines lines and $expectedNumberOfFileSources file sources',
+ 'Filtering logs on $logLevelFilters level should return $expectedNumberOfLines lines and $expectedNumberOfFileSources file sources',
({
- logLevelFilter,
+ logLevelFilters,
expectedNumberOfLines, expectedNumberOfFileSources,
}) => {
const { parsedLogs, fileSources } = parseLogs(
mockTaskLog,
null,
- logLevelFilter,
- null,
+ logLevelFilters,
+ [],
);
expect(fileSources).toHaveLength(expectedNumberOfFileSources);
- const lines = parsedLogs.split('\n');
+ expect(parsedLogs).toBeDefined();
+ const lines = parsedLogs!.split('\n');
expect(lines).toHaveLength(expectedNumberOfLines);
- lines.forEach((line) => expect(line).toContain(logLevelFilter));
+ lines.forEach((line) => expect(line).toContain(logLevelFilters[0]));
},
);
@@ -94,8 +99,8 @@ describe('Test Logs Utils.', () => {
const { parsedLogs, fileSources } = parseLogs(
mockTaskLog,
null,
- null,
- 'taskinstance.py',
+ [],
+ ['taskinstance.py'],
);
expect(fileSources).toEqual([
@@ -104,7 +109,7 @@ describe('Test Logs Utils.', () => {
'task_command.py',
'taskinstance.py',
]);
- const lines = parsedLogs.split('\n');
+ const lines = parsedLogs!.split('\n');
expect(lines).toHaveLength(7);
lines.forEach((line) => expect(line).toContain('taskinstance.py'));
});
@@ -113,8 +118,8 @@ describe('Test Logs Utils.', () => {
const { parsedLogs, fileSources } = parseLogs(
mockTaskLog,
null,
- 'INFO',
- 'taskinstance.py',
+ [LogLevel.INFO, LogLevel.WARNING],
+ ['taskinstance.py'],
);
expect(fileSources).toEqual([
@@ -123,8 +128,8 @@ describe('Test Logs Utils.', () => {
'task_command.py',
'taskinstance.py',
]);
- const lines = parsedLogs.split('\n');
- expect(lines).toHaveLength(6);
- lines.forEach((line) => expect(line).toContain('INFO'));
+ const lines = parsedLogs!.split('\n');
+ expect(lines).toHaveLength(7);
+ lines.forEach((line) => expect(line).toMatch(/INFO|WARNING/));
});
});
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.ts
similarity index 78%
rename from airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js
rename to airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.ts
index 0678f520a2..14b9cb8bf9 100644
--- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js
+++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.ts
@@ -21,29 +21,42 @@
import { defaultFormatWithTZ } from '../../../../../datetime_utils';
-export const logLevel = {
- DEBUG: 'DEBUG',
- INFO: 'INFO',
- WARNING: 'WARNING',
- ERROR: 'ERROR',
- CRITICAL: 'CRITICAL',
-};
+export enum LogLevel {
+ DEBUG = 'DEBUG',
+ INFO = 'INFO',
+ WARNING = 'WARNING',
+ ERROR = 'ERROR',
+ CRITICAL = 'CRITICAL',
+}
-export const parseLogs = (data, timezone, logLevelFilter, fileSourceFilter) => {
- const lines = data.split('\n');
+export const logLevelColorMapping = {
+ [LogLevel.DEBUG]: 'gray.300',
+ [LogLevel.INFO]: 'green.200',
+ [LogLevel.WARNING]: 'yellow.200',
+ [LogLevel.ERROR]: 'red.200',
+ [LogLevel.CRITICAL]: 'red.400',
+};
+export const parseLogs = (
+ data: string | undefined,
+ timezone: string | null,
+ logLevelFilters: Array<LogLevel>,
+ fileSourceFilters: Array<string>,
+) => {
if (!data) {
return {};
}
- const parsedLines = [];
- const fileSources = new Set();
+ const lines = data.split('\n');
+
+ const parsedLines: Array<string> = [];
+ const fileSources: Set<string> = new Set();
lines.forEach((line) => {
let parsedLine = line;
// Apply log level filter.
- if (logLevelFilter && !line.includes(logLevelFilter)) {
+ if (logLevelFilters.length > 0 && logLevelFilters.every((level) => !line.includes(level))) {
return;
}
@@ -67,6 +80,7 @@ export const parseLogs = (data, timezone, logLevelFilter, fileSourceFilter) => {
// keep previous behavior if utcoffset not found. (consider it UTC)
//
if (dateTime && timezone) { // dateTime === fullMatch
+ // @ts-ignore
const localDateTime = moment.utc(dateTime).tz(timezone).format(defaultFormatWithTZ);
parsedLine = line.replace(dateTime, localDateTime);
}
@@ -76,6 +90,7 @@ export const parseLogs = (data, timezone, logLevelFilter, fileSourceFilter) => {
const [utcoffset, threeDigitMs] = msecOrUTCOffset.split(' ');
const msec = threeDigitMs.replace(/\D+/g, ''); // drop 'ms'
// e.g) datetime='2022-06-15 10:30:06.123+0900'
+ // @ts-ignore
const localDateTime = moment(`${date}.${msec}${utcoffset}`).tz(timezone).format(defaultFormatWithTZ);
parsedLine = line.replace(dateTime, localDateTime);
}
@@ -83,7 +98,9 @@ export const parseLogs = (data, timezone, logLevelFilter, fileSourceFilter) => {
[logGroup] = matches[2].split(':');
fileSources.add(logGroup);
}
- if (!fileSourceFilter || fileSourceFilter === logGroup) {
+
+ if (fileSourceFilters.length === 0
+ || fileSourceFilters.some((fileSourceFilter) => line.includes(fileSourceFilter))) {
parsedLines.push(parsedLine);
}
});
diff --git a/airflow/www/static/js/grid/details/content/taskInstance/index.tsx b/airflow/www/static/js/grid/details/content/taskInstance/index.tsx
index bbb262eeda..9acaee7481 100644
--- a/airflow/www/static/js/grid/details/content/taskInstance/index.tsx
+++ b/airflow/www/static/js/grid/details/content/taskInstance/index.tsx
@@ -49,7 +49,7 @@ import type { Task, DagRun } from '../../../types';
const detailsPanelActiveTabIndex = 'detailsPanelActiveTabIndex';
-const dagId = getMetaValue('dag_id');
+const dagId = getMetaValue('dag_id')!;
interface Props {
taskId: Task['id'];
@@ -205,7 +205,7 @@ const TaskInstance = ({ taskId, runId }: Props) => {
<Logs
dagId={dagId}
dagRunId={runId}
- taskId={taskId}
+ taskId={taskId!}
executionDate={executionDate}
tryNumber={instance?.tryNumber}
/>
diff --git a/airflow/www/static/js/grid/types/index.ts b/airflow/www/static/js/grid/types/index.ts
index df1f873ef7..71d67f8862 100644
--- a/airflow/www/static/js/grid/types/index.ts
+++ b/airflow/www/static/js/grid/types/index.ts
@@ -32,6 +32,15 @@ type TaskState = RunState
| 'deferred'
| null;
+interface Dag {
+ id: string,
+ rootDagId: string,
+ isPaused: boolean,
+ isSubdag: boolean,
+ owners: Array<string>,
+ description: string,
+}
+
interface DagRun {
runId: string;
runType: 'manual' | 'backfill' | 'scheduled';
@@ -67,6 +76,7 @@ interface Task {
}
export type {
+ Dag,
DagRun,
RunState,
TaskState,
diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock
index edeb3eb35e..887b5cb25c 100644
--- a/airflow/www/yarn.lock
+++ b/airflow/www/yarn.lock
@@ -1326,6 +1326,13 @@
dependencies:
regenerator-runtime "^0.13.4"
+"@babel/runtime@^7.12.0", "@babel/runtime@^7.16.3", "@babel/runtime@^7.8.7":
+ version "7.18.3"
+ resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.3.tgz#c7b654b57f6f63cf7f8b418ac9ca04408c4579f4"
+ integrity sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug==
+ dependencies:
+ regenerator-runtime "^0.13.4"
+
"@babel/runtime@^7.12.5", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
version "7.16.0"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.16.0.tgz#e27b977f2e2088ba24748bf99b5e1dece64e4f0b"
@@ -1347,13 +1354,6 @@
dependencies:
regenerator-runtime "^0.13.4"
-"@babel/runtime@^7.16.3":
- version "7.18.3"
- resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.3.tgz#c7b654b57f6f63cf7f8b418ac9ca04408c4579f4"
- integrity sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug==
- dependencies:
- regenerator-runtime "^0.13.4"
-
"@babel/runtime@^7.7.6":
version "7.17.9"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.17.9.tgz#d19fbf802d01a8cb6cf053a64e472d42c434ba72"
@@ -2077,7 +2077,7 @@
source-map "^0.5.7"
stylis "4.0.13"
-"@emotion/cache@^11.9.3":
+"@emotion/cache@^11.4.0", "@emotion/cache@^11.9.3":
version "11.9.3"
resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.9.3.tgz#96638449f6929fd18062cfe04d79b29b44c0d6cb"
integrity sha512-0dgkI/JKlCXa+lEXviaMtGBL0ynpx4osh7rjOXE71q9bIF8G+XhJgvi+wDu0B0IdCVx37BffiwXlN9I3UuzFvg==
@@ -2117,7 +2117,7 @@
resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.5.tgz#2c40f81449a4e554e9fc6396910ed4843ec2be50"
integrity sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ==
-"@emotion/react@^11.9.3":
+"@emotion/react@^11.8.1", "@emotion/react@^11.9.3":
version "11.9.3"
resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.9.3.tgz#f4f4f34444f6654a2e550f5dab4f2d360c101df9"
integrity sha512-g9Q1GcTOlzOEjqwuLF/Zd9LC+4FljjPjDfxSM7KmEakm+hsHXk+bYZ2q+/hTJzr0OUNkujo72pXLQvXj6H+GJQ==
@@ -2817,6 +2817,13 @@
dependencies:
"@types/react" "*"
+"@types/react-transition-group@^4.4.0":
+ version "4.4.5"
+ resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.5.tgz#aae20dcf773c5aa275d5b9f7cdbca638abc5e416"
+ integrity sha512-juKD/eiSM3/xZYzjuzH6ZwpP+/lejltmiS3QEzV/vmb/Q8+HfDmxu+Baga8UEMGBqV88Nbg4l2hY/K2DkyaLLA==
+ dependencies:
+ "@types/react" "*"
+
"@types/react@*":
version "17.0.34"
resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.34.tgz#797b66d359b692e3f19991b6b07e4b0c706c0102"
@@ -3785,6 +3792,13 @@ caniuse-lite@^1.0.30001349:
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001354.tgz#95c5efdb64148bb4870771749b9a619304755ce5"
integrity sha512-mImKeCkyGDAHNywYFA4bqnLAzTUvVkqPvhY4DV47X+Gl2c5Z8c3KNETnXp14GQt11LvxE8AwjzGxJ+rsikiOzg==
+chakra-react-select@^4.0.0:
+ version "4.0.3"
+ resolved "https://registry.yarnpkg.com/chakra-react-select/-/chakra-react-select-4.0.3.tgz#6760a92ee0b814ec89181503dde796584360e03d"
+ integrity sha512-QEjySGsd666s0LSrLxpJiOv0mVFPVHVjPMcj3JRga3H/rHpUukZ6ydYX0uXl0WMZtUST7R9hcKNs0bzA6RTP8Q==
+ dependencies:
+ react-select "^5.3.2"
+
chalk@^2.0.0:
version "2.4.2"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -4757,6 +4771,14 @@ dom-accessibility-api@^0.5.6, dom-accessibility-api@^0.5.9:
resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.10.tgz#caa6d08f60388d0bb4539dd75fe458a9a1d0014c"
integrity sha512-Xu9mD0UjrJisTmv7lmVSDMagQcU9R5hwAbxsaAE/35XPnPLJobbuREfV/rraiSaEj/UOvgrzQs66zyTWTlyd+g==
+dom-helpers@^5.0.1:
+ version "5.2.1"
+ resolved "https://registry.yarnpkg.com/dom-helpers/-/dom-helpers-5.2.1.tgz#d9400536b2bf8225ad98fe052e029451ac40e902"
+ integrity sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==
+ dependencies:
+ "@babel/runtime" "^7.8.7"
+ csstype "^3.0.2"
+
dom-serializer@0:
version "0.2.2"
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51"
@@ -7230,6 +7252,11 @@ mdn-data@2.0.14:
resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50"
integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==
+memoize-one@^5.0.0:
+ version "5.2.1"
+ resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e"
+ integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==
+
meow@^9.0.0:
version "9.0.0"
resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364"
@@ -8394,7 +8421,7 @@ prop-types@^15.5.0:
object-assign "^4.1.1"
react-is "^16.8.1"
-prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1:
+prop-types@^15.6.0, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1:
version "15.8.1"
resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
@@ -8537,6 +8564,19 @@ react-router@6.3.0:
dependencies:
history "^5.2.0"
+react-select@^5.3.2:
+ version "5.3.2"
+ resolved "https://registry.yarnpkg.com/react-select/-/react-select-5.3.2.tgz#ecee0d5c59ed4acb7f567f7de3c75a488d93dacb"
+ integrity sha512-W6Irh7U6Ha7p5uQQ2ZnemoCQ8mcfgOtHfw3wuMzG6FAu0P+CYicgofSLOq97BhjMx8jS+h+wwWdCBeVVZ9VqlQ==
+ dependencies:
+ "@babel/runtime" "^7.12.0"
+ "@emotion/cache" "^11.4.0"
+ "@emotion/react" "^11.8.1"
+ "@types/react-transition-group" "^4.4.0"
+ memoize-one "^5.0.0"
+ prop-types "^15.6.0"
+ react-transition-group "^4.3.0"
+
react-style-singleton@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4"
@@ -8559,6 +8599,16 @@ react-tabs@^3.2.2:
clsx "^1.1.0"
prop-types "^15.5.0"
+react-transition-group@^4.3.0:
+ version "4.4.2"
+ resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.2.tgz#8b59a56f09ced7b55cbd53c36768b922890d5470"
+ integrity sha512-/RNYfRAMlZwDSr6z4zNKV6xu53/e2BuaBbGhbyYIXTrmgu/bGHzmqOs7mJSJBHy9Ud+ApHx3QjrkKSp1pxvlFg==
+ dependencies:
+ "@babel/runtime" "^7.5.5"
+ dom-helpers "^5.0.1"
+ loose-envify "^1.4.0"
+ prop-types "^15.6.2"
+
react@^18.0.0:
version "18.1.0"
resolved "https://registry.yarnpkg.com/react/-/react-18.1.0.tgz#6f8620382decb17fdc5cc223a115e2adbf104890"