You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2023/02/06 21:51:05 UTC
[airflow] branch main updated: Fix grid logs for large logs (#29390)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1ed08028ef Fix grid logs for large logs (#29390)
1ed08028ef is described below
commit 1ed08028efec9284c490f98df5ce9ba355b673ff
Author: Brent Bovenzi <br...@astronomer.io>
AuthorDate: Mon Feb 6 18:50:52 2023 -0300
Fix grid logs for large logs (#29390)
* slice extra long file lines, remove full content
* add warning if logs are truncated
* Add warning for `.split()` errors
* remove extraneous useTaskLog change
* remove Show Full Content checkbox
* remove extraneous console.error
---
airflow/www/static/js/api/useTaskLog.ts | 2 +-
.../dag/details/taskInstance/Logs/index.test.tsx | 35 ----------------------
.../js/dag/details/taskInstance/Logs/index.tsx | 25 +++++++++-------
.../js/dag/details/taskInstance/Logs/utils.ts | 21 +++++++++++--
4 files changed, 34 insertions(+), 49 deletions(-)
diff --git a/airflow/www/static/js/api/useTaskLog.ts b/airflow/www/static/js/api/useTaskLog.ts
index bbb6395878..53e0e57ca4 100644
--- a/airflow/www/static/js/api/useTaskLog.ts
+++ b/airflow/www/static/js/api/useTaskLog.ts
@@ -32,7 +32,7 @@ interface Props extends API.GetLogVariables {
}
const useTaskLog = ({
- dagId, dagRunId, taskId, taskTryNumber, mapIndex, fullContent, state,
+ dagId, dagRunId, taskId, taskTryNumber, mapIndex, fullContent = false, state,
}: Props) => {
let url: string = '';
const [isPreviousStatePending, setPrevState] = useState(true);
diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx b/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx
index 10d1c48863..19fd3ee953 100644
--- a/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx
+++ b/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx
@@ -81,7 +81,6 @@ describe('Test Logs Component.', () => {
expect(useTaskLogMock).toHaveBeenLastCalledWith({
dagId: 'dummyDagId',
dagRunId: 'dummyDagRunId',
- fullContent: false,
taskId: 'dummyTaskId',
taskTryNumber: 2,
});
@@ -141,7 +140,6 @@ describe('Test Logs Component.', () => {
expect(useTaskLogMock).toHaveBeenLastCalledWith({
dagId: 'dummyDagId',
dagRunId: 'dummyDagRunId',
- fullContent: false,
mapIndex: 1,
taskId: 'dummyTaskId',
taskTryNumber: 2,
@@ -168,7 +166,6 @@ describe('Test Logs Component.', () => {
expect(useTaskLogMock).toHaveBeenLastCalledWith({
dagId: 'dummyDagId',
dagRunId: 'dummyDagRunId',
- fullContent: false,
taskId: 'dummyTaskId',
taskTryNumber: 2,
});
@@ -179,40 +176,8 @@ describe('Test Logs Component.', () => {
expect(useTaskLogMock).toHaveBeenLastCalledWith({
dagId: 'dummyDagId',
dagRunId: 'dummyDagRunId',
- fullContent: false,
taskId: 'dummyTaskId',
taskTryNumber: 1,
});
});
-
- test('Test Logs Full Content', () => {
- const tryNumber = 2;
- const { getByTestId } = render(
- <Logs
- dagId="dummyDagId"
- dagRunId="dummyDagRunId"
- taskId="dummyTaskId"
- executionDate="2020:01:01T01:00+00:00"
- tryNumber={tryNumber}
- />,
- );
- expect(useTaskLogMock).toHaveBeenLastCalledWith({
- dagId: 'dummyDagId',
- dagRunId: 'dummyDagRunId',
- fullContent: false,
- taskId: 'dummyTaskId',
- taskTryNumber: 2,
- });
- const fullContentCheckbox = getByTestId('full-content-checkbox');
-
- fireEvent.click(fullContentCheckbox);
-
- expect(useTaskLogMock).toHaveBeenLastCalledWith({
- dagId: 'dummyDagId',
- dagRunId: 'dummyDagRunId',
- fullContent: true,
- taskId: 'dummyTaskId',
- taskTryNumber: 2,
- });
- });
});
diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx b/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx
index 0139d8e223..5ebf7b2a98 100644
--- a/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx
+++ b/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx
@@ -27,7 +27,9 @@ import {
Divider,
Button,
Checkbox,
+ Icon,
} from '@chakra-ui/react';
+import { MdWarning } from 'react-icons/md';
import { getMetaValue } from 'src/utils';
import useTaskLog from 'src/api/useTaskLog';
@@ -102,7 +104,6 @@ const Logs = ({
}: Props) => {
const [internalIndexes, externalIndexes] = getLinkIndexes(tryNumber);
const [selectedTryNumber, setSelectedTryNumber] = useState<number | undefined>();
- const [shouldRequestFullContent, setShouldRequestFullContent] = useState(false);
const [wrap, setWrap] = useState(getMetaValue('default_wrap') === 'True');
const [logLevelFilters, setLogLevelFilters] = useState<Array<LogLevelOption>>([]);
const [fileSourceFilters, setFileSourceFilters] = useState<Array<FileSourceOption>>([]);
@@ -115,7 +116,6 @@ const Logs = ({
taskId,
mapIndex,
taskTryNumber,
- fullContent: shouldRequestFullContent,
state,
});
@@ -128,7 +128,11 @@ const Logs = ({
params.append('map_index', mapIndex.toString());
}
- const { parsedLogs, fileSources = [] } = useMemo(
+ const {
+ parsedLogs,
+ fileSources = [],
+ warning,
+ } = useMemo(
() => parseLogs(
data,
timezone,
@@ -222,13 +226,6 @@ const Logs = ({
>
<Text as="strong">Wrap</Text>
</Checkbox>
- <Checkbox
- onChange={() => setShouldRequestFullContent((previousState) => !previousState)}
- px={4}
- data-testid="full-content-checkbox"
- >
- <Text as="strong" whiteSpace="nowrap">Full Logs</Text>
- </Checkbox>
<LogLink
dagId={dagId}
taskId={taskId}
@@ -245,6 +242,14 @@ const Logs = ({
</Flex>
</Flex>
</Box>
+ {!!warning && (
+ <Flex bg="yellow.200" borderRadius={2} borderColor="gray.400" alignItems="center" p={2}>
+ <Icon as={MdWarning} color="yellow.500" mr={2} />
+ <Text fontSize="sm">
+ {warning}
+ </Text>
+ </Flex>
+ )}
{!!parsedLogs && (
<LogBlock
parsedLogs={parsedLogs}
diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts
index c340e5253e..75e7ca1bed 100644
--- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts
+++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts
@@ -47,11 +47,14 @@ export const parseLogs = (
return {};
}
let lines;
+
+ let warning;
+
try {
lines = data.split('\n');
} catch (err) {
- console.error(err);
- return {};
+ warning = 'Unable to show logs. There was an error parsing logs.';
+ return { warning };
}
const parsedLines: Array<string> = [];
@@ -87,5 +90,17 @@ export const parseLogs = (
}
});
- return { parsedLogs: parsedLines.join('\n'), fileSources: Array.from(fileSources).sort() };
+ return {
+ parsedLogs: parsedLines
+ .map((l) => {
+ if (l.length >= 1000000) {
+ warning = 'Large log file. Some lines have been truncated. Download logs in order to see everything.';
+ return `${l.slice(0, 1000000)}...`;
+ }
+ return l;
+ })
+ .join('\n'),
+ fileSources: Array.from(fileSources).sort(),
+ warning,
+ };
};