You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/09/11 12:15:25 UTC

[airflow] 01/01: dbt, openlineage: set run_id after defer, do not log error if operator has no run_id set

This is an automated email from the ASF dual-hosted git repository.

mobuchowski pushed a commit to branch openlineage-fix-dbt-no-run-id-on-defer
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 7afe067bb07549a41d5ced113178a297089b94cd
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Mon Sep 11 14:14:19 2023 +0200

    dbt, openlineage: set run_id after defer, do not log error if operator has no run_id set
    
    Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
 airflow/providers/dbt/cloud/operators/dbt.py        | 7 ++++---
 tests/providers/dbt/cloud/utils/test_openlineage.py | 8 ++++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/airflow/providers/dbt/cloud/operators/dbt.py b/airflow/providers/dbt/cloud/operators/dbt.py
index a4c04a0919..c0b37c7fdd 100644
--- a/airflow/providers/dbt/cloud/operators/dbt.py
+++ b/airflow/providers/dbt/cloud/operators/dbt.py
@@ -117,7 +117,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         self.timeout = timeout
         self.check_interval = check_interval
         self.additional_run_config = additional_run_config or {}
-        self.run_id: int
+        self.run_id: int | None = None
         self.deferrable = deferrable
 
     def execute(self, context: Context):
@@ -140,7 +140,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         # run can be monitored via the operator link.
         context["ti"].xcom_push(key="job_run_url", value=job_run_url)
 
-        if self.wait_for_termination:
+        if self.wait_for_termination and isinstance(self.run_id, int):
             if self.deferrable is False:
                 self.log.info("Waiting for job run %s to terminate.", str(self.run_id))
 
@@ -197,6 +197,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         if event["status"] == "error":
             raise AirflowException(event["message"])
         self.log.info(event["message"])
+        self.run_id = event["run_id"]
         return int(event["run_id"])
 
     def on_kill(self) -> None:
@@ -225,7 +226,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         """
         from airflow.providers.openlineage.extractors import OperatorLineage
 
-        if self.wait_for_termination is True:
+        if isinstance(self.run_id, int) and self.wait_for_termination is True:
             return generate_openlineage_events_from_dbt_cloud_run(operator=self, task_instance=task_instance)
         return OperatorLineage()
 
diff --git a/tests/providers/dbt/cloud/utils/test_openlineage.py b/tests/providers/dbt/cloud/utils/test_openlineage.py
index 0d93d5a2e1..ed9c7bfa40 100644
--- a/tests/providers/dbt/cloud/utils/test_openlineage.py
+++ b/tests/providers/dbt/cloud/utils/test_openlineage.py
@@ -22,6 +22,7 @@ from unittest.mock import MagicMock, patch
 from airflow.providers.dbt.cloud.hooks.dbt import DbtCloudHook
 from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator
 from airflow.providers.dbt.cloud.utils.openlineage import generate_openlineage_events_from_dbt_cloud_run
+from airflow.providers.openlineage.extractors import OperatorLineage
 
 TASK_ID = "dbt_test"
 DAG_ID = "dbt_dag"
@@ -130,7 +131,10 @@ class TestGenerateOpenLineageEventsFromDbtCloudRun:
         )
 
         mock_build_task_instance_run_id.return_value = TASK_UUID
-
         generate_openlineage_events_from_dbt_cloud_run(mock_operator, task_instance=mock_task_instance)
-
         assert mock_client.emit.call_count == 4
+
+    def test_do_not_raise_error_if_runid_not_set_on_operator(self):
+        operator = DbtCloudRunJobOperator(task_id="dbt-job-runid-taskid", job_id=1500)
+        assert operator.run_id is None
+        assert operator.get_openlineage_facets_on_complete(MagicMock()) == OperatorLineage()