You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bb...@apache.org on 2022/10/06 18:42:44 UTC
[airflow] branch main updated: Add search to datasets list (#26893)
This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9514b6ca70 Add search to datasets list (#26893)
9514b6ca70 is described below
commit 9514b6ca701728bc21c609af18a59a2ba9cffe97
Author: Brent Bovenzi <br...@astronomer.io>
AuthorDate: Thu Oct 6 14:42:33 2022 -0400
Add search to datasets list (#26893)
* add search to datasets list
* include search in url params
---
airflow/www/static/js/datasets/List.test.tsx | 119 +++++++++++++++++++++++++++
airflow/www/static/js/datasets/List.tsx | 46 ++++++++++-
airflow/www/views.py | 2 +
tests/www/views/test_views_dataset.py | 29 +++++++
4 files changed, 192 insertions(+), 4 deletions(-)
diff --git a/airflow/www/static/js/datasets/List.test.tsx b/airflow/www/static/js/datasets/List.test.tsx
new file mode 100644
index 0000000000..6dad8c7a04
--- /dev/null
+++ b/airflow/www/static/js/datasets/List.test.tsx
@@ -0,0 +1,119 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* global describe, test, expect */
+
+import React from 'react';
+import { render } from '@testing-library/react';
+
+import * as useDatasetsModule from 'src/api/useDatasets';
+import { Wrapper } from 'src/utils/testUtils';
+
+import DatasetsList from './List';
+
+const datasets = [
+ {
+ id: 0,
+ uri: 'this_dataset',
+ extra: null,
+ lastDatasetUpdate: null,
+ totalUpdates: 0,
+ createdAt: new Date().toISOString(),
+ updatedAt: new Date().toISOString(),
+ },
+ {
+ id: 1,
+ uri: 'that_dataset',
+ extra: null,
+ lastDatasetUpdate: new Date().toISOString(),
+ totalUpdates: 10,
+ createdAt: new Date().toISOString(),
+ updatedAt: new Date().toISOString(),
+ },
+ {
+ id: 1,
+ uri: 'extra_dataset',
+ extra: null,
+ lastDatasetUpdate: new Date().toISOString(),
+ totalUpdates: 1,
+ createdAt: new Date().toISOString(),
+ updatedAt: new Date().toISOString(),
+ },
+];
+
+const returnValue = {
+ data: {
+ datasets,
+ totalEntries: datasets.length,
+ },
+ isSuccess: true,
+} as any;
+
+const emptyReturnValue = {
+ data: {
+ datasets: [],
+ totalEntries: 0,
+ },
+ isSuccess: true,
+ isLoading: false,
+} as any;
+
+describe('Test Datasets List', () => {
+ test('Displays a list of datasets', () => {
+ jest.spyOn(useDatasetsModule, 'default').mockImplementation(() => returnValue);
+
+ const { getByText, queryAllByTestId } = render(
+ <DatasetsList
+ onSelect={() => {}}
+ />,
+ { wrapper: Wrapper },
+ );
+
+ const listItems = queryAllByTestId('dataset-list-item');
+
+ expect(listItems).toHaveLength(3);
+
+ expect(getByText(datasets[0].uri)).toBeDefined();
+ expect(getByText('Total Updates: 0')).toBeDefined();
+
+ expect(getByText(datasets[1].uri)).toBeDefined();
+ expect(getByText('Total Updates: 10')).toBeDefined();
+
+ expect(getByText(datasets[2].uri)).toBeDefined();
+ expect(getByText('Total Updates: 1')).toBeDefined();
+ });
+
+ test('Empty state displays when there are no datasets', () => {
+ jest.spyOn(useDatasetsModule, 'default').mockImplementation(() => emptyReturnValue);
+
+ const { getByText, queryAllByTestId, getByTestId } = render(
+ <DatasetsList
+ onSelect={() => {}}
+ />,
+ { wrapper: Wrapper },
+ );
+
+ const listItems = queryAllByTestId('dataset-list-item');
+
+ expect(listItems).toHaveLength(0);
+
+ expect(getByTestId('no-datasets-msg')).toBeInTheDocument();
+ expect(getByText('No Data found.')).toBeInTheDocument();
+ });
+});
diff --git a/airflow/www/static/js/datasets/List.tsx b/airflow/www/static/js/datasets/List.tsx
index 8ac1bb7622..eed417f1b8 100644
--- a/airflow/www/static/js/datasets/List.tsx
+++ b/airflow/www/static/js/datasets/List.tsx
@@ -24,14 +24,21 @@ import {
Flex,
Text,
Link,
+ Input,
+ InputGroup,
+ InputLeftElement,
+ InputRightElement,
+ IconButton,
} from '@chakra-ui/react';
+import { snakeCase } from 'lodash';
import type { Row, SortingRule } from 'react-table';
+import { MdClose, MdSearch } from 'react-icons/md';
+import { useSearchParams } from 'react-router-dom';
import { useDatasets } from 'src/api';
import { Table, TimeCell } from 'src/components/Table';
import type { API } from 'src/types';
import { getMetaValue } from 'src/utils';
-import { snakeCase } from 'lodash';
interface Props {
onSelect: (datasetId: string) => void;
@@ -49,7 +56,7 @@ interface CellProps {
const DetailCell = ({ cell: { row } }: CellProps) => {
const { totalUpdates, uri } = row.original;
return (
- <Box>
+ <Box data-testid="dataset-list-item">
<Text>{uri}</Text>
<Text fontSize="sm" mt={2}>
Total Updates:
@@ -60,18 +67,24 @@ const DetailCell = ({ cell: { row } }: CellProps) => {
);
};
+const SEARCH_PARAM = 'search';
+
const DatasetsList = ({ onSelect }: Props) => {
const limit = 25;
const [offset, setOffset] = useState(0);
+ const [searchParams, setSearchParams] = useSearchParams();
+ const search = searchParams.get(SEARCH_PARAM) || '';
const [sortBy, setSortBy] = useState<SortingRule<object>[]>([{ id: 'lastDatasetUpdate', desc: true }]);
const sort = sortBy[0];
const order = sort ? `${sort.desc ? '-' : ''}${snakeCase(sort.id)}` : '';
+ const uri = search.length > 2 ? search : undefined;
const { data: { datasets, totalEntries }, isLoading } = useDatasets({
limit,
offset,
order,
+ uri,
});
const columns = useMemo(
@@ -102,6 +115,16 @@ const DatasetsList = ({ onSelect }: Props) => {
const docsUrl = getMetaValue('datasets_docs');
+ const onSearch = (e: React.ChangeEvent<HTMLInputElement>) => {
+ searchParams.set(SEARCH_PARAM, encodeURIComponent(e.target.value));
+ setSearchParams(searchParams);
+ };
+
+ const onClear = () => {
+ searchParams.delete(SEARCH_PARAM);
+ setSearchParams(searchParams);
+ };
+
return (
<Box>
<Flex justifyContent="space-between" alignItems="center">
@@ -109,8 +132,8 @@ const DatasetsList = ({ onSelect }: Props) => {
Datasets
</Heading>
</Flex>
- {!datasets.length && !isLoading && (
- <Text mb={4}>
+ {!datasets.length && !isLoading && !search && (
+ <Text mb={4} data-testid="no-datasets-msg">
Looks like you do not have any datasets yet. Check out the
{' '}
<Link color="blue" href={docsUrl} isExternal>docs</Link>
@@ -118,6 +141,21 @@ const DatasetsList = ({ onSelect }: Props) => {
to learn how to create a dataset.
</Text>
)}
+ <InputGroup my={2} px={1}>
+ <InputLeftElement pointerEvents="none">
+ <MdSearch />
+ </InputLeftElement>
+ <Input
+ placeholder="Search by URI..."
+ value={search}
+ onChange={onSearch}
+ />
+ {search.length > 0 && (
+ <InputRightElement>
+ <IconButton aria-label="Clear search" title="Clear search" icon={<MdClose />} variant="ghost" onClick={onClear} />
+ </InputRightElement>
+ )}
+ </InputGroup>
<Box borderWidth={1}>
<Table
data={data}
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 79d0262aba..67f7b327dd 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -3524,6 +3524,7 @@ class Airflow(AirflowBaseView):
limit = int(request.args.get("limit", 25))
offset = int(request.args.get("offset", 0))
order_by = request.args.get("order_by", "uri")
+ uri_pattern = request.args.get("uri_pattern", "")
lstripped_orderby = order_by.lstrip('-')
if lstripped_orderby not in allowed_attrs:
@@ -3573,6 +3574,7 @@ class Airflow(AirflowBaseView):
DatasetModel.id,
DatasetModel.uri,
)
+ .filter(DatasetModel.uri.ilike(f"%{uri_pattern}%"))
.order_by(*order_by)
.offset(offset)
.limit(limit)
diff --git a/tests/www/views/test_views_dataset.py b/tests/www/views/test_views_dataset.py
index c67298876d..a5eb6e23f4 100644
--- a/tests/www/views/test_views_dataset.py
+++ b/tests/www/views/test_views_dataset.py
@@ -131,6 +131,35 @@ class TestGetDatasets(TestDatasetEndpoint):
assert ordered_dataset_ids == [json_dict['id'] for json_dict in response.json['datasets']]
assert response.json['total_entries'] == len(ordered_dataset_ids)
+ def test_search_uri_pattern(self, admin_client, session):
+ datasets = [
+ DatasetModel(
+ id=i,
+ uri=f"s3://bucket/key_{i}",
+ )
+ for i in [1, 2]
+ ]
+ session.add_all(datasets)
+ session.commit()
+ assert session.query(DatasetModel).count() == 2
+
+ uri_pattern = 'key_2'
+ response = admin_client.get(f"/object/datasets_summary?uri_pattern={uri_pattern}")
+
+ assert response.status_code == 200
+ response_data = response.json
+ assert response_data == {
+ "datasets": [
+ {
+ "id": 2,
+ "uri": "s3://bucket/key_2",
+ "last_dataset_update": None,
+ "total_updates": 0,
+ },
+ ],
+ "total_entries": 2,
+ }
+
@pytest.mark.need_serialized_dag
def test_correct_counts_update(self, admin_client, session, dag_maker, app, monkeypatch):
with monkeypatch.context() as m: