You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/02/11 22:28:49 UTC
[airflow] branch main updated: Rename `db export-cleaned` to `db export-archived` (#29450)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0b9dc1a806 Rename `db export-cleaned` to `db export-archived` (#29450)
0b9dc1a806 is described below
commit 0b9dc1a806f3818c07934b8b7d6a15c56ed9874d
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Sat Feb 11 23:28:41 2023 +0100
Rename `db export-cleaned` to `db export-archived` (#29450)
* Rename `db export-cleaned` to `db export-archived`
This is more appropriate because what is exported are the contents of the archived tables.
Even though the contents are the cleaned data, they are still archived and we are
'exporting' from the archived tables.
This also aligns with the `--drop-archives` option and `db drop-archived` command in terms of naming.
* fixup! Rename `db export-cleaned` to `db export-archived`
---
airflow/cli/cli_parser.py | 6 +++---
airflow/cli/commands/db_command.py | 6 +++---
airflow/utils/db_cleanup.py | 4 ++--
docs/apache-airflow/howto/usage-cli.rst | 4 ++--
tests/cli/commands/test_db_command.py | 18 +++++++++---------
tests/cli/test_cli_parser.py | 10 +++++-----
tests/utils/test_db_cleanup.py | 14 +++++++-------
7 files changed, 31 insertions(+), 31 deletions(-)
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
index 08abbcfe37..344a5b07fb 100644
--- a/airflow/cli/cli_parser.py
+++ b/airflow/cli/cli_parser.py
@@ -1616,9 +1616,9 @@ DB_COMMANDS = (
),
),
ActionCommand(
- name="export-cleaned",
- help="Export cleaned data from the archive tables",
- func=lazy_load_command("airflow.cli.commands.db_command.export_cleaned"),
+ name="export-archived",
+ help="Export archived data from the archive tables",
+ func=lazy_load_command("airflow.cli.commands.db_command.export_archived"),
args=(
ARG_DB_EXPORT_FORMAT,
ARG_DB_OUTPUT_PATH,
diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py
index 72ee55c86d..e80e2d27bc 100644
--- a/airflow/cli/commands/db_command.py
+++ b/airflow/cli/commands/db_command.py
@@ -27,7 +27,7 @@ from airflow import settings
from airflow.exceptions import AirflowException
from airflow.utils import cli as cli_utils, db
from airflow.utils.db import REVISION_HEADS_MAP
-from airflow.utils.db_cleanup import config_dict, drop_archived_tables, export_cleaned_records, run_cleanup
+from airflow.utils.db_cleanup import config_dict, drop_archived_tables, export_archived_records, run_cleanup
from airflow.utils.process_utils import execute_interactive
@@ -210,9 +210,9 @@ def cleanup_tables(args):
@cli_utils.action_cli(check_db=False)
-def export_cleaned(args):
+def export_archived(args):
"""Exports archived records from metadata database."""
- export_cleaned_records(
+ export_archived_records(
export_format=args.export_format,
output_path=args.output_path,
table_names=args.tables,
diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py
index 41b89931f5..e2b6b5d1d2 100644
--- a/airflow/utils/db_cleanup.py
+++ b/airflow/utils/db_cleanup.py
@@ -428,10 +428,10 @@ def run_cleanup(
@provide_session
-def export_cleaned_records(
+def export_archived_records(
export_format, output_path, table_names=None, drop_archives=False, session: Session = NEW_SESSION
):
- """Export cleaned data to the given output path in the given format."""
+ """Export archived data to the given output path in the given format."""
archived_table_names = _get_archived_table_names(table_names, session)
# If user chose to drop archives, check there are archive tables that exists
# before asking for confirmation
diff --git a/docs/apache-airflow/howto/usage-cli.rst b/docs/apache-airflow/howto/usage-cli.rst
index cd66d7e0cb..ec403350f2 100644
--- a/docs/apache-airflow/howto/usage-cli.rst
+++ b/docs/apache-airflow/howto/usage-cli.rst
@@ -219,7 +219,7 @@ By default, ``db clean`` will archive purged rows in tables of the form ``_airfl
Export the purged records from the archive tables
-------------------------------------------------
-The ``db export-cleaned`` command exports the contents of the archived tables, created by the ``db clean`` command,
+The ``db export-archived`` command exports the contents of the archived tables, created by the ``db clean`` command,
to a specified format, by default to a CSV file. The exported file will contain the records that were purged from the
primary tables during the ``db clean`` process.
@@ -237,7 +237,7 @@ Dropping the archived tables
If during the ``db clean`` process, you did not use the ``--skip-archive`` option which drops the archived table, you can
still drop the archive tables using the ``db drop-archived`` command. This operation is irreversible and you are encouraged
-to use the ``db export-cleaned`` command to backup the tables to disk before dropping them.
+to use the ``db export-archived`` command to backup the tables to disk before dropping them.
You can specify the tables to drop using the ``--tables`` option. If no tables are specified, all archive tables will be
dropped.
diff --git a/tests/cli/commands/test_db_command.py b/tests/cli/commands/test_db_command.py
index f91a7767ad..fa1e34d730 100644
--- a/tests/cli/commands/test_db_command.py
+++ b/tests/cli/commands/test_db_command.py
@@ -449,18 +449,18 @@ class TestCLIDBClean:
skip_archive=False,
)
- @patch("airflow.cli.commands.db_command.export_cleaned_records")
+ @patch("airflow.cli.commands.db_command.export_archived_records")
@patch("airflow.cli.commands.db_command.os.path.isdir", return_value=True)
def test_export_archived_records(self, os_mock, export_archived_mock):
args = self.parser.parse_args(
[
"db",
- "export-cleaned",
+ "export-archived",
"--output-path",
"path",
]
)
- db_command.export_cleaned(args)
+ db_command.export_archived(args)
export_archived_mock.assert_called_once_with(
export_format="csv", output_path="path", table_names=None, drop_archives=False
@@ -469,7 +469,7 @@ class TestCLIDBClean:
@pytest.mark.parametrize(
"extra_args, expected", [(["--tables", "hello, goodbye"], ["hello", "goodbye"]), ([], None)]
)
- @patch("airflow.cli.commands.db_command.export_cleaned_records")
+ @patch("airflow.cli.commands.db_command.export_archived_records")
@patch("airflow.cli.commands.db_command.os.path.isdir", return_value=True)
def test_tables_in_export_archived_records_command(
self, os_mock, export_archived_mock, extra_args, expected
@@ -477,19 +477,19 @@ class TestCLIDBClean:
args = self.parser.parse_args(
[
"db",
- "export-cleaned",
+ "export-archived",
"--output-path",
"path",
*extra_args,
]
)
- db_command.export_cleaned(args)
+ db_command.export_archived(args)
export_archived_mock.assert_called_once_with(
export_format="csv", output_path="path", table_names=expected, drop_archives=False
)
@pytest.mark.parametrize("extra_args, expected", [(["--drop-archives"], True), ([], False)])
- @patch("airflow.cli.commands.db_command.export_cleaned_records")
+ @patch("airflow.cli.commands.db_command.export_archived_records")
@patch("airflow.cli.commands.db_command.os.path.isdir", return_value=True)
def test_drop_archives_in_export_archived_records_command(
self, os_mock, export_archived_mock, extra_args, expected
@@ -497,13 +497,13 @@ class TestCLIDBClean:
args = self.parser.parse_args(
[
"db",
- "export-cleaned",
+ "export-archived",
"--output-path",
"path",
*extra_args,
]
)
- db_command.export_cleaned(args)
+ db_command.export_archived(args)
export_archived_mock.assert_called_once_with(
export_format="csv", output_path="path", table_names=None, drop_archives=expected
)
diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py
index 8c9a09c55b..41143bebe9 100644
--- a/tests/cli/test_cli_parser.py
+++ b/tests/cli/test_cli_parser.py
@@ -242,17 +242,17 @@ class TestCli:
with contextlib.redirect_stderr(io.StringIO()) as stderr:
with pytest.raises(SystemExit):
parser = cli_parser.get_parser()
- parser.parse_args(["db", "export-cleaned", "--output-path", "/non/existing/directory"])
+ parser.parse_args(["db", "export-archived", "--output-path", "/non/existing/directory"])
error_msg = stderr.getvalue()
assert error_msg == (
- "\nairflow db export-cleaned command error: The directory "
+ "\nairflow db export-archived command error: The directory "
"'/non/existing/directory' does not exist!, see help above.\n"
)
@pytest.mark.parametrize("export_format", ["json", "yaml", "unknown"])
@patch("airflow.cli.cli_parser.os.path.isdir", return_value=True)
- def test_invalid_choice_raises_for_export_format_in_db_export_cleaned_command(
+ def test_invalid_choice_raises_for_export_format_in_db_export_archived_command(
self, mock_isdir, export_format
):
"""Test that invalid choice raises for export-format in db export-cleaned command."""
@@ -260,11 +260,11 @@ class TestCli:
with pytest.raises(SystemExit):
parser = cli_parser.get_parser()
parser.parse_args(
- ["db", "export-cleaned", "--export-format", export_format, "--output-path", "mydir"]
+ ["db", "export-archived", "--export-format", export_format, "--output-path", "mydir"]
)
error_msg = stderr.getvalue()
assert error_msg == (
- "\nairflow db export-cleaned command error: argument "
+ "\nairflow db export-archived command error: argument "
f"--export-format: invalid choice: '{export_format}' "
"(choose from 'csv'), see help above.\n"
)
diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py
index 7d36e37ced..f9c34d54fe 100644
--- a/tests/utils/test_db_cleanup.py
+++ b/tests/utils/test_db_cleanup.py
@@ -43,7 +43,7 @@ from airflow.utils.db_cleanup import (
_dump_table_to_file,
config_dict,
drop_archived_tables,
- export_cleaned_records,
+ export_archived_records,
run_cleanup,
)
from airflow.utils.session import create_session
@@ -346,7 +346,7 @@ class TestDBCleanup:
"""test that drop confirmation input is called when appropriate"""
inspector = inspect_mock.return_value
inspector.get_table_names.return_value = [f"{ARCHIVE_TABLE_PREFIX}dag_run__233"]
- export_cleaned_records(
+ export_archived_records(
export_format="csv", output_path="path", drop_archives=drop_archive, session=MagicMock()
)
if drop_archive:
@@ -398,14 +398,14 @@ class TestDBCleanup:
@patch("airflow.utils.db_cleanup._dump_table_to_file")
@patch("airflow.utils.db_cleanup.inspect")
@patch("builtins.input", side_effect=["drop archived tables"])
- def test_export_cleaned_records_only_archived_tables(
+ def test_export_archived_records_only_archived_tables(
self, mock_input, inspect_mock, dump_mock, caplog, drop_archive
):
- """Test export_cleaned_records and show that only tables with the archive prefix are exported."""
+ """Test export_archived_records and show that only tables with the archive prefix are exported."""
session_mock = MagicMock()
inspector = inspect_mock.return_value
inspector.get_table_names.return_value = [f"{ARCHIVE_TABLE_PREFIX}dag_run__233", "task_instance"]
- export_cleaned_records(
+ export_archived_records(
export_format="csv", output_path="path", drop_archives=drop_archive, session=session_mock
)
dump_mock.assert_called_once_with(
@@ -426,7 +426,7 @@ class TestDBCleanup:
@patch("airflow.utils.db_cleanup.inspect")
@patch("airflow.utils.db_cleanup._confirm_drop_archives")
@patch("builtins.input", side_effect=["drop archived tables"])
- def test_export_cleaned_no_confirm_if_no_tables(
+ def test_export_archived_no_confirm_if_no_tables(
self, mock_input, mock_confirm, inspect_mock, dump_mock, caplog, drop_archive
):
"""Test no confirmation if no archived tables found"""
@@ -434,7 +434,7 @@ class TestDBCleanup:
inspector = inspect_mock.return_value
# No tables with the archive prefix
inspector.get_table_names.return_value = ["dag_run", "task_instance"]
- export_cleaned_records(
+ export_archived_records(
export_format="csv", output_path="path", drop_archives=drop_archive, session=session_mock
)
mock_confirm.assert_not_called()