[airflow] branch main updated: Add option to show output of `SQLExecuteQueryOperator` in the log (#29954)

[el...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new a9b79a27b2 Add option to show output of `SQLExecuteQueryOperator` in the log (#29954)
a9b79a27b2 is described below

commit a9b79a27b25a47c7e0390c139b517f229fdacd12
Author: eladkal <45...@users.noreply.github.com>
AuthorDate: Wed Mar 8 06:17:13 2023 +0200

    Add option to show output of `SQLExecuteQueryOperator` in the log (#29954)
    
    * Add option to show output of `SQLExecuteQueryOperator` in the log
---
 airflow/providers/common/sql/operators/sql.py  | 6 ++++++
 airflow/providers/common/sql/operators/sql.pyi | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/airflow/providers/common/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py
index 824732f3d9..24397592ea 100644
--- a/airflow/providers/common/sql/operators/sql.py
+++ b/airflow/providers/common/sql/operators/sql.py
@@ -203,6 +203,8 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
     :param split_statements: (optional) if split single SQL string into statements. By default, defers
         to the default value in the ``run`` method of the configured hook.
     :param return_last: (optional) return the result of only last statement (default: True).
+    :param show_return_value_in_logs: (optional) if true operator output will be printed to the task log.
+        Use with caution. It's not recommended to dump large datasets to the log. (default: False).
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -223,6 +225,7 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
         handler: Callable[[Any], Any] = fetch_all_handler,
         split_statements: bool | None = None,
         return_last: bool = True,
+        show_return_value_in_logs: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -232,6 +235,7 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
         self.handler = handler
         self.split_statements = split_statements
         self.return_last = return_last
+        self.show_return_value_in_logs = show_return_value_in_logs
 
     def _process_output(self, results: list[Any], descriptions: list[Sequence[Sequence] | None]) -> list[Any]:
         """
@@ -250,6 +254,8 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
         :param results: results in the form of list of rows.
         :param descriptions: list of descriptions returned by ``cur.description`` in the Python DBAPI
         """
+        if self.show_return_value_in_logs:
+            self.log.info("Operator output is: %s", results)
         return results
 
     def execute(self, context):
diff --git a/airflow/providers/common/sql/operators/sql.pyi b/airflow/providers/common/sql/operators/sql.pyi
index 72d77a0e6a..f2735b4a25 100644
--- a/airflow/providers/common/sql/operators/sql.pyi
+++ b/airflow/providers/common/sql/operators/sql.pyi
@@ -68,6 +68,7 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
     handler: Incomplete
     split_statements: Incomplete
     return_last: Incomplete
+    show_return_value_in_logs: Incomplete
     def __init__(
         self,
         *,
@@ -77,6 +78,7 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
         handler: Callable[[Any], Any] = ...,
         split_statements: Union[bool, None] = ...,
         return_last: bool = ...,
+        show_return_value_in_logs: bool = ...,
         **kwargs,
     ) -> None: ...
     def execute(self, context): ...