You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/12/23 13:38:00 UTC
[airflow] branch master updated: Disable suppress_logs_and_warning
in cli when debugging (#13180)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 40d2d6b Disable suppress_logs_and_warning in cli when debugging (#13180)
40d2d6b is described below
commit 40d2d6bc157ca8ff9e633d07782ccfe6a489bf3b
Author: Tomek Urbaszek <tu...@gmail.com>
AuthorDate: Wed Dec 23 13:37:38 2020 +0000
Disable suppress_logs_and_warning in cli when debugging (#13180)
* Disable suppress_logs_and_warning in cli when debugging
In some cases commands like 'dags list' can be used for debug purposes.
The problem is that we are suppresing logs and warnings in some cases
to make the output nice and clean. This commit disable this functionality
if logging level is set to DEBUG.
* Add verbose flag
* fixup! Add verbose flag
---
airflow/cli/cli_parser.py | 73 +++++++++++++++--------------
airflow/cli/commands/cheat_sheet_command.py | 2 +-
airflow/cli/commands/connection_command.py | 4 +-
airflow/cli/commands/dag_command.py | 8 ++--
airflow/cli/commands/info_command.py | 2 +-
airflow/cli/commands/plugins_command.py | 2 +-
airflow/cli/commands/pool_command.py | 10 ++--
airflow/cli/commands/provider_command.py | 12 ++---
airflow/cli/commands/role_command.py | 4 +-
airflow/cli/commands/task_command.py | 2 +-
airflow/cli/commands/user_command.py | 2 +-
airflow/cli/commands/variable_command.py | 4 +-
airflow/utils/cli.py | 50 +++++++++++++-------
13 files changed, 96 insertions(+), 79 deletions(-)
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
index efcab74..35949ea 100644
--- a/airflow/cli/cli_parser.py
+++ b/airflow/cli/cli_parser.py
@@ -721,13 +721,13 @@ DAGS_COMMANDS = (
name='list',
help="List all the DAGs",
func=lazy_load_command('airflow.cli.commands.dag_command.dag_list_dags'),
- args=(ARG_SUBDIR, ARG_OUTPUT),
+ args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='report',
help='Show DagBag loading report',
func=lazy_load_command('airflow.cli.commands.dag_command.dag_report'),
- args=(ARG_SUBDIR, ARG_OUTPUT),
+ args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='list-runs',
@@ -740,13 +740,21 @@ DAGS_COMMANDS = (
"all the dagruns that were executed after this date. "
),
func=lazy_load_command('airflow.cli.commands.dag_command.dag_list_dag_runs'),
- args=(ARG_DAG_ID_OPT, ARG_NO_BACKFILL, ARG_STATE, ARG_OUTPUT, ARG_START_DATE, ARG_END_DATE),
+ args=(
+ ARG_DAG_ID_OPT,
+ ARG_NO_BACKFILL,
+ ARG_STATE,
+ ARG_OUTPUT,
+ ARG_VERBOSE,
+ ARG_START_DATE,
+ ARG_END_DATE,
+ ),
),
ActionCommand(
name='list-jobs',
help="List the jobs",
func=lazy_load_command('airflow.cli.commands.dag_command.dag_list_jobs'),
- args=(ARG_DAG_ID_OPT, ARG_STATE, ARG_LIMIT, ARG_OUTPUT),
+ args=(ARG_DAG_ID_OPT, ARG_STATE, ARG_LIMIT, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='state',
@@ -980,7 +988,7 @@ TASKS_COMMANDS = (
name='states-for-dag-run',
help="Get the status of all task instances in a dag run",
func=lazy_load_command('airflow.cli.commands.task_command.task_states_for_dag_run'),
- args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_OUTPUT),
+ args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_OUTPUT, ARG_VERBOSE),
),
)
POOLS_COMMANDS = (
@@ -988,42 +996,31 @@ POOLS_COMMANDS = (
name='list',
help='List pools',
func=lazy_load_command('airflow.cli.commands.pool_command.pool_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='get',
help='Get pool size',
func=lazy_load_command('airflow.cli.commands.pool_command.pool_get'),
- args=(
- ARG_POOL_NAME,
- ARG_OUTPUT,
- ),
+ args=(ARG_POOL_NAME, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='set',
help='Configure pool',
func=lazy_load_command('airflow.cli.commands.pool_command.pool_set'),
- args=(
- ARG_POOL_NAME,
- ARG_POOL_SLOTS,
- ARG_POOL_DESCRIPTION,
- ARG_OUTPUT,
- ),
+ args=(ARG_POOL_NAME, ARG_POOL_SLOTS, ARG_POOL_DESCRIPTION, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='delete',
help='Delete pool',
func=lazy_load_command('airflow.cli.commands.pool_command.pool_delete'),
- args=(
- ARG_POOL_NAME,
- ARG_OUTPUT,
- ),
+ args=(ARG_POOL_NAME, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='import',
help='Import pools',
func=lazy_load_command('airflow.cli.commands.pool_command.pool_import'),
- args=(ARG_POOL_IMPORT,),
+ args=(ARG_POOL_IMPORT, ARG_VERBOSE),
),
ActionCommand(
name='export',
@@ -1037,13 +1034,13 @@ VARIABLES_COMMANDS = (
name='list',
help='List variables',
func=lazy_load_command('airflow.cli.commands.variable_command.variables_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='get',
help='Get variable',
func=lazy_load_command('airflow.cli.commands.variable_command.variables_get'),
- args=(ARG_VAR, ARG_JSON, ARG_DEFAULT),
+ args=(ARG_VAR, ARG_JSON, ARG_DEFAULT, ARG_VERBOSE),
),
ActionCommand(
name='set',
@@ -1114,13 +1111,13 @@ CONNECTIONS_COMMANDS = (
name='get',
help='Get a connection',
func=lazy_load_command('airflow.cli.commands.connection_command.connections_get'),
- args=(ARG_CONN_ID, ARG_COLOR, ARG_OUTPUT),
+ args=(ARG_CONN_ID, ARG_COLOR, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='list',
help='List connections',
func=lazy_load_command('airflow.cli.commands.connection_command.connections_list'),
- args=(ARG_OUTPUT, ARG_CONN_ID_FILTER),
+ args=(ARG_OUTPUT, ARG_VERBOSE, ARG_CONN_ID_FILTER),
),
ActionCommand(
name='add',
@@ -1161,37 +1158,40 @@ PROVIDERS_COMMANDS = (
name='list',
help='List installed providers',
func=lazy_load_command('airflow.cli.commands.provider_command.providers_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='get',
help='Get detailed information about a provider',
func=lazy_load_command('airflow.cli.commands.provider_command.provider_get'),
- args=(ARG_OUTPUT, ARG_FULL, ARG_COLOR, ARG_PROVIDER_NAME),
+ args=(ARG_OUTPUT, ARG_VERBOSE, ARG_FULL, ARG_COLOR, ARG_PROVIDER_NAME),
),
ActionCommand(
name='links',
help='List extra links registered by the providers',
func=lazy_load_command('airflow.cli.commands.provider_command.extra_links_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='widgets',
help='Get information about registered connection form widgets',
func=lazy_load_command('airflow.cli.commands.provider_command.connection_form_widget_list'),
- args=(ARG_OUTPUT,),
+ args=(
+ ARG_OUTPUT,
+ ARG_VERBOSE,
+ ),
),
ActionCommand(
name='hooks',
help='List registered provider hooks',
func=lazy_load_command('airflow.cli.commands.provider_command.hooks_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='behaviours',
help='Get information about registered connection types with custom behaviours',
func=lazy_load_command('airflow.cli.commands.provider_command.connection_field_behaviours'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
)
@@ -1200,7 +1200,7 @@ USERS_COMMANDS = (
name='list',
help='List users',
func=lazy_load_command('airflow.cli.commands.user_command.users_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='create',
@@ -1263,13 +1263,13 @@ ROLES_COMMANDS = (
name='list',
help='List roles',
func=lazy_load_command('airflow.cli.commands.role_command.roles_list'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name='create',
help='Create role',
func=lazy_load_command('airflow.cli.commands.role_command.roles_create'),
- args=(ARG_ROLES,),
+ args=(ARG_ROLES, ARG_VERBOSE),
),
)
@@ -1443,7 +1443,7 @@ airflow_commands: List[CLICommand] = [
name='cheat-sheet',
help="Display cheat sheet",
func=lazy_load_command('airflow.cli.commands.cheat_sheet_command.cheat_sheet'),
- args=(),
+ args=(ARG_VERBOSE,),
),
GroupCommand(
name='connections',
@@ -1490,13 +1490,14 @@ airflow_commands: List[CLICommand] = [
args=(
ARG_ANONYMIZE,
ARG_FILE_IO,
+ ARG_VERBOSE,
),
),
ActionCommand(
name='plugins',
help='Dump information about loaded plugins',
func=lazy_load_command('airflow.cli.commands.plugins_command.dump_plugins'),
- args=(ARG_OUTPUT,),
+ args=(ARG_OUTPUT, ARG_VERBOSE),
),
GroupCommand(
name="celery",
diff --git a/airflow/cli/commands/cheat_sheet_command.py b/airflow/cli/commands/cheat_sheet_command.py
index 263688d..e77ebad 100644
--- a/airflow/cli/commands/cheat_sheet_command.py
+++ b/airflow/cli/commands/cheat_sheet_command.py
@@ -25,7 +25,7 @@ from airflow.utils.cli import suppress_logs_and_warning
from airflow.utils.helpers import partition
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def cheat_sheet(args):
"""Display cheat-sheet."""
display_commands_index()
diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py
index 202a252..6e8bdfb 100644
--- a/airflow/cli/commands/connection_command.py
+++ b/airflow/cli/commands/connection_command.py
@@ -52,7 +52,7 @@ def _connection_mapper(conn: Connection) -> Dict[str, Any]:
}
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def connections_get(args):
"""Get a connection."""
try:
@@ -66,7 +66,7 @@ def connections_get(args):
)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def connections_list(args):
"""Lists all connections at the command line"""
with create_session() as session:
diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py
index 573f70a..0ecdfef 100644
--- a/airflow/cli/commands/dag_command.py
+++ b/airflow/cli/commands/dag_command.py
@@ -269,7 +269,7 @@ def dag_next_execution(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def dag_list_dags(args):
"""Displays dags with or without stats at the command line"""
dagbag = DagBag(process_subdir(args.subdir))
@@ -286,7 +286,7 @@ def dag_list_dags(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def dag_report(args):
"""Displays dagbag stats at the command line"""
dagbag = DagBag(process_subdir(args.subdir))
@@ -304,7 +304,7 @@ def dag_report(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def dag_list_jobs(args, dag=None):
"""Lists latest n jobs"""
queries = []
@@ -339,7 +339,7 @@ def dag_list_jobs(args, dag=None):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def dag_list_dag_runs(args, dag=None):
"""Lists dag runs for a given DAG"""
if dag:
diff --git a/airflow/cli/commands/info_command.py b/airflow/cli/commands/info_command.py
index 202c61b..9a32520 100644
--- a/airflow/cli/commands/info_command.py
+++ b/airflow/cli/commands/info_command.py
@@ -413,7 +413,7 @@ def _send_report_to_fileio(info):
print(str(ex))
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def show_info(args):
"""Show information related to Airflow, system and other."""
# Enforce anonymization, when file_io upload is tuned on.
diff --git a/airflow/cli/commands/plugins_command.py b/airflow/cli/commands/plugins_command.py
index 42a35a8..a53f0c9 100644
--- a/airflow/cli/commands/plugins_command.py
+++ b/airflow/cli/commands/plugins_command.py
@@ -49,7 +49,7 @@ def _join_plugins_names(value: Union[List[Any], Any]) -> str:
return ",".join(_get_name(v) for v in value)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def dump_plugins(args):
"""Dump plugins information"""
plugins_manager.ensure_plugins_loaded()
diff --git a/airflow/cli/commands/pool_command.py b/airflow/cli/commands/pool_command.py
index d61b116..b116e25 100644
--- a/airflow/cli/commands/pool_command.py
+++ b/airflow/cli/commands/pool_command.py
@@ -39,7 +39,7 @@ def _show_pools(pools, output):
)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def pool_list(args):
"""Displays info of all the pools"""
api_client = get_current_api_client()
@@ -47,7 +47,7 @@ def pool_list(args):
_show_pools(pools=pools, output=args.output)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def pool_get(args):
"""Displays pool info by a given name"""
api_client = get_current_api_client()
@@ -59,7 +59,7 @@ def pool_get(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def pool_set(args):
"""Creates new pool with a given name and slots"""
api_client = get_current_api_client()
@@ -68,7 +68,7 @@ def pool_set(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def pool_delete(args):
"""Deletes pool by a given name"""
api_client = get_current_api_client()
@@ -80,7 +80,7 @@ def pool_delete(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def pool_import(args):
"""Imports pools from the file"""
if not os.path.exists(args.file):
diff --git a/airflow/cli/commands/provider_command.py b/airflow/cli/commands/provider_command.py
index 6738654..65f3a48 100644
--- a/airflow/cli/commands/provider_command.py
+++ b/airflow/cli/commands/provider_command.py
@@ -26,7 +26,7 @@ def _remove_rst_syntax(value: str) -> str:
return re.sub("[`_<>]", "", value.strip(" \n."))
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def provider_get(args):
"""Get a provider info."""
providers = ProvidersManager().providers
@@ -46,7 +46,7 @@ def provider_get(args):
raise SystemExit(f"No such provider installed: {args.provider_name}")
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def providers_list(args):
"""Lists all providers at the command line"""
AirflowConsole().print_as(
@@ -60,7 +60,7 @@ def providers_list(args):
)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def hooks_list(args):
"""Lists all hooks at the command line"""
AirflowConsole().print_as(
@@ -76,7 +76,7 @@ def hooks_list(args):
)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def connection_form_widget_list(args):
"""Lists all custom connection form fields at the command line"""
AirflowConsole().print_as(
@@ -91,7 +91,7 @@ def connection_form_widget_list(args):
)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def connection_field_behaviours(args):
"""Lists field behaviours"""
AirflowConsole().print_as(
@@ -103,7 +103,7 @@ def connection_field_behaviours(args):
)
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def extra_links_list(args):
"""Lists all extra links at the command line"""
AirflowConsole().print_as(
diff --git a/airflow/cli/commands/role_command.py b/airflow/cli/commands/role_command.py
index f9fb701..b017cd1 100644
--- a/airflow/cli/commands/role_command.py
+++ b/airflow/cli/commands/role_command.py
@@ -24,7 +24,7 @@ from airflow.utils.cli import suppress_logs_and_warning
from airflow.www.app import cached_app
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def roles_list(args):
"""Lists all existing roles"""
appbuilder = cached_app().appbuilder # pylint: disable=no-member
@@ -35,7 +35,7 @@ def roles_list(args):
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def roles_create(args):
"""Creates new empty role in DB"""
appbuilder = cached_app().appbuilder # pylint: disable=no-member
diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py
index b153d20..c3bce77 100644
--- a/airflow/cli/commands/task_command.py
+++ b/airflow/cli/commands/task_command.py
@@ -314,7 +314,7 @@ def _guess_debugger():
@cli_utils.action_logging
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def task_states_for_dag_run(args):
"""Get the status of all task instances in a DagRun"""
with create_session() as session:
diff --git a/airflow/cli/commands/user_command.py b/airflow/cli/commands/user_command.py
index 3fd80bd..12d18a0 100644
--- a/airflow/cli/commands/user_command.py
+++ b/airflow/cli/commands/user_command.py
@@ -29,7 +29,7 @@ from airflow.utils.cli import suppress_logs_and_warning
from airflow.www.app import cached_app
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def users_list(args):
"""Lists users at the command line"""
appbuilder = cached_app().appbuilder # pylint: disable=no-member
diff --git a/airflow/cli/commands/variable_command.py b/airflow/cli/commands/variable_command.py
index 55cf94b..8783d61 100644
--- a/airflow/cli/commands/variable_command.py
+++ b/airflow/cli/commands/variable_command.py
@@ -27,7 +27,7 @@ from airflow.utils.cli import suppress_logs_and_warning
from airflow.utils.session import create_session
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def variables_list(args):
"""Displays all of the variables"""
with create_session() as session:
@@ -35,7 +35,7 @@ def variables_list(args):
AirflowConsole().print_as(data=variables, output=args.output, mapper=lambda x: {"key": x.key})
-@suppress_logs_and_warning()
+@suppress_logs_and_warning
def variables_get(args):
"""Displays variable by a given name"""
try:
diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py
index 68a0b44..d6f8c54 100644
--- a/airflow/utils/cli.py
+++ b/airflow/utils/cli.py
@@ -17,7 +17,6 @@
# under the License.
#
"""Utilities module for cli"""
-import contextlib
import functools
import getpass
import json
@@ -45,6 +44,15 @@ if TYPE_CHECKING:
from airflow.models import DAG
+def _check_cli_args(args):
+ if not args:
+ raise ValueError("Args should be set")
+ if not isinstance(args[0], Namespace):
+ raise ValueError(
+ "1st positional argument should be argparse.Namespace instance," f"but is {type(args[0])}"
+ )
+
+
def action_logging(f: T) -> T:
"""
Decorates function to execute function at the same time submitting action_logging
@@ -77,12 +85,7 @@ def action_logging(f: T) -> T:
at 1st positional argument
:param kwargs: A passthrough keyword argument
"""
- if not args:
- raise ValueError("Args should be set")
- if not isinstance(args[0], Namespace):
- raise ValueError(
- "1st positional argument should be argparse.Namespace instance," f"but is {type(args[0])}"
- )
+ _check_cli_args(args)
metrics = _build_metrics(f.__name__, args[0])
cli_action_loggers.on_pre_execution(**metrics)
try:
@@ -290,13 +293,26 @@ def should_use_colors(args) -> bool:
return is_terminal_support_colors()
-@contextlib.contextmanager
-def suppress_logs_and_warning():
- """Context manager to suppress logging and warning messages"""
- with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- logging.disable(logging.CRITICAL)
- yield
- # logging output again depends on the effective
- # levels of individual loggers
- logging.disable(logging.NOTSET)
+def suppress_logs_and_warning(f: T) -> T:
+ """
+ Decorator to suppress logging and warning messages
+ in cli functions.
+ """
+
+ @functools.wraps(f)
+ def _wrapper(*args, **kwargs):
+ _check_cli_args(args)
+ if args[0].verbose:
+ f(*args, **kwargs)
+ else:
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ logging.disable(logging.CRITICAL)
+ try:
+ f(*args, **kwargs)
+ finally:
+ # logging output again depends on the effective
+ # levels of individual loggers
+ logging.disable(logging.NOTSET)
+
+ return cast(T, _wrapper)