You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2024/10/21 11:05:17 UTC
(airflow) branch processing-facet-dag-event created (now 8a2f077327)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch processing-facet-dag-event
in repository https://gitbox.apache.org/repos/asf/airflow.git
at 8a2f077327 add ProcessingEngineRunFacet to OL DAG Start event
This branch includes the following new commits:
new 8a2f077327 add ProcessingEngineRunFacet to OL DAG Start event
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow) 01/01: add ProcessingEngineRunFacet to OL DAG Start event
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch processing-facet-dag-event
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8a2f0773278b551c2efedeb5feb0eabe08ca6d74
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Mon Oct 21 13:04:38 2024 +0200
add ProcessingEngineRunFacet to OL DAG Start event
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
.../src/airflow/providers/openlineage/plugins/adapter.py | 14 +++-----------
.../src/airflow/providers/openlineage/utils/utils.py | 16 ++++++++++++++--
providers/tests/openlineage/plugins/test_adapter.py | 3 +++
3 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/providers/src/airflow/providers/openlineage/plugins/adapter.py b/providers/src/airflow/providers/openlineage/plugins/adapter.py
index 51eb649adb..daa154693a 100644
--- a/providers/src/airflow/providers/openlineage/plugins/adapter.py
+++ b/providers/src/airflow/providers/openlineage/plugins/adapter.py
@@ -32,7 +32,6 @@ from openlineage.client.facet_v2 import (
nominal_time_run,
ownership_job,
parent_run,
- processing_engine_run,
source_code_location_job,
)
from openlineage.client.uuid import generate_static_uuid
@@ -42,6 +41,7 @@ from airflow.providers.openlineage.utils.utils import (
OpenLineageRedactor,
get_airflow_debug_facet,
get_airflow_state_run_facet,
+ get_processing_engine_facet,
)
from airflow.stats import Stats
from airflow.utils.log.logging_mixin import LoggingMixin
@@ -197,18 +197,10 @@ class OpenLineageAdapter(LoggingMixin):
:param task: metadata container with information extracted from operator
:param run_facets: custom run facets
"""
- from airflow.version import version as AIRFLOW_VERSION
-
- processing_engine_version_facet = processing_engine_run.ProcessingEngineRunFacet(
- version=AIRFLOW_VERSION,
- name="Airflow",
- openlineageAdapterVersion=OPENLINEAGE_PROVIDER_VERSION,
- )
-
run_facets = run_facets or {}
if task:
run_facets = {**task.run_facets, **run_facets}
- run_facets["processing_engine"] = processing_engine_version_facet # type: ignore
+ run_facets = {**run_facets, **get_processing_engine_facet()} # type: ignore
event = RunEvent(
eventType=RunState.START,
eventTime=event_time,
@@ -364,7 +356,7 @@ class OpenLineageAdapter(LoggingMixin):
job_name=dag_id,
nominal_start_time=nominal_start_time,
nominal_end_time=nominal_end_time,
- run_facets={**run_facets, **get_airflow_debug_facet()},
+ run_facets={**run_facets, **get_airflow_debug_facet(), **get_processing_engine_facet()},
),
inputs=[],
outputs=[],
diff --git a/providers/src/airflow/providers/openlineage/utils/utils.py b/providers/src/airflow/providers/openlineage/utils/utils.py
index 3d05d564a7..786c58ff12 100644
--- a/providers/src/airflow/providers/openlineage/utils/utils.py
+++ b/providers/src/airflow/providers/openlineage/utils/utils.py
@@ -38,7 +38,7 @@ from airflow.exceptions import (
# TODO: move this maybe to Airflow's logic?
from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
from airflow.providers.common.compat.assets import Asset
-from airflow.providers.openlineage import conf
+from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
from airflow.providers.openlineage.plugins.facets import (
AirflowDagRunFacet,
AirflowDebugRunFacet,
@@ -65,7 +65,7 @@ from airflow.utils.module_loading import import_string
if TYPE_CHECKING:
from openlineage.client.event_v2 import Dataset as OpenLineageDataset
- from openlineage.client.facet_v2 import RunFacet
+ from openlineage.client.facet_v2 import RunFacet, processing_engine_run
from airflow.models import TaskInstance
from airflow.utils.state import DagRunState, TaskInstanceState
@@ -406,6 +406,18 @@ def _get_all_packages_installed() -> dict[str, str]:
return {dist.metadata["Name"]: dist.version for dist in metadata.distributions()}
+def get_processing_engine_facet() -> dict[str, processing_engine_run.ProcessingEngineRunFacet]:
+ from openlineage.client.facet_v2 import processing_engine_run
+
+ return {
+ "processing_engine": processing_engine_run.ProcessingEngineRunFacet(
+ version=AIRFLOW_VERSION,
+ name="Airflow",
+ openlineageAdapterVersion=OPENLINEAGE_PROVIDER_VERSION,
+ )
+ }
+
+
def get_airflow_debug_facet() -> dict[str, AirflowDebugRunFacet]:
if not conf.debug_mode():
return {}
diff --git a/providers/tests/openlineage/plugins/test_adapter.py b/providers/tests/openlineage/plugins/test_adapter.py
index a2af9f393d..154a9b1c29 100644
--- a/providers/tests/openlineage/plugins/test_adapter.py
+++ b/providers/tests/openlineage/plugins/test_adapter.py
@@ -617,6 +617,9 @@ def test_emit_dag_started_event(mock_stats_incr, mock_stats_timer, generate_stat
nominalStartTime=event_time.isoformat(),
nominalEndTime=event_time.isoformat(),
),
+ "processing_engine": processing_engine_run.ProcessingEngineRunFacet(
+ version=ANY, name="Airflow", openlineageAdapterVersion=ANY
+ ),
"airflowDagRun": AirflowDagRunFacet(
dag=expected_dag_info,
dagRun={