You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/09/11 12:15:25 UTC
[airflow] 01/01: dbt, openlineage: set run_id after defer, do not log error if operator has no run_id set
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch openlineage-fix-dbt-no-run-id-on-defer
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7afe067bb07549a41d5ced113178a297089b94cd
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Mon Sep 11 14:14:19 2023 +0200
dbt, openlineage: set run_id after defer, do not log error if operator has no run_id set
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
airflow/providers/dbt/cloud/operators/dbt.py | 7 ++++---
tests/providers/dbt/cloud/utils/test_openlineage.py | 8 ++++++--
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/airflow/providers/dbt/cloud/operators/dbt.py b/airflow/providers/dbt/cloud/operators/dbt.py
index a4c04a0919..c0b37c7fdd 100644
--- a/airflow/providers/dbt/cloud/operators/dbt.py
+++ b/airflow/providers/dbt/cloud/operators/dbt.py
@@ -117,7 +117,7 @@ class DbtCloudRunJobOperator(BaseOperator):
self.timeout = timeout
self.check_interval = check_interval
self.additional_run_config = additional_run_config or {}
- self.run_id: int
+ self.run_id: int | None = None
self.deferrable = deferrable
def execute(self, context: Context):
@@ -140,7 +140,7 @@ class DbtCloudRunJobOperator(BaseOperator):
# run can be monitored via the operator link.
context["ti"].xcom_push(key="job_run_url", value=job_run_url)
- if self.wait_for_termination:
+ if self.wait_for_termination and isinstance(self.run_id, int):
if self.deferrable is False:
self.log.info("Waiting for job run %s to terminate.", str(self.run_id))
@@ -197,6 +197,7 @@ class DbtCloudRunJobOperator(BaseOperator):
if event["status"] == "error":
raise AirflowException(event["message"])
self.log.info(event["message"])
+ self.run_id = event["run_id"]
return int(event["run_id"])
def on_kill(self) -> None:
@@ -225,7 +226,7 @@ class DbtCloudRunJobOperator(BaseOperator):
"""
from airflow.providers.openlineage.extractors import OperatorLineage
- if self.wait_for_termination is True:
+ if isinstance(self.run_id, int) and self.wait_for_termination is True:
return generate_openlineage_events_from_dbt_cloud_run(operator=self, task_instance=task_instance)
return OperatorLineage()
diff --git a/tests/providers/dbt/cloud/utils/test_openlineage.py b/tests/providers/dbt/cloud/utils/test_openlineage.py
index 0d93d5a2e1..ed9c7bfa40 100644
--- a/tests/providers/dbt/cloud/utils/test_openlineage.py
+++ b/tests/providers/dbt/cloud/utils/test_openlineage.py
@@ -22,6 +22,7 @@ from unittest.mock import MagicMock, patch
from airflow.providers.dbt.cloud.hooks.dbt import DbtCloudHook
from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator
from airflow.providers.dbt.cloud.utils.openlineage import generate_openlineage_events_from_dbt_cloud_run
+from airflow.providers.openlineage.extractors import OperatorLineage
TASK_ID = "dbt_test"
DAG_ID = "dbt_dag"
@@ -130,7 +131,10 @@ class TestGenerateOpenLineageEventsFromDbtCloudRun:
)
mock_build_task_instance_run_id.return_value = TASK_UUID
-
generate_openlineage_events_from_dbt_cloud_run(mock_operator, task_instance=mock_task_instance)
-
assert mock_client.emit.call_count == 4
+
+ def test_do_not_raise_error_if_runid_not_set_on_operator(self):
+ operator = DbtCloudRunJobOperator(task_id="dbt-job-runid-taskid", job_id=1500)
+ assert operator.run_id is None
+ assert operator.get_openlineage_facets_on_complete(MagicMock()) == OperatorLineage()