You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2024/10/21 11:05:17 UTC

(airflow) branch processing-facet-dag-event created (now 8a2f077327)

This is an automated email from the ASF dual-hosted git repository.

mobuchowski pushed a change to branch processing-facet-dag-event
in repository https://gitbox.apache.org/repos/asf/airflow.git


      at 8a2f077327 add ProcessingEngineRunFacet to OL DAG Start event

This branch includes the following new commits:

     new 8a2f077327 add ProcessingEngineRunFacet to OL DAG Start event

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(airflow) 01/01: add ProcessingEngineRunFacet to OL DAG Start event

Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mobuchowski pushed a commit to branch processing-facet-dag-event
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 8a2f0773278b551c2efedeb5feb0eabe08ca6d74
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Mon Oct 21 13:04:38 2024 +0200

    add ProcessingEngineRunFacet to OL DAG Start event
    
    Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
 .../src/airflow/providers/openlineage/plugins/adapter.py | 14 +++-----------
 .../src/airflow/providers/openlineage/utils/utils.py     | 16 ++++++++++++++--
 providers/tests/openlineage/plugins/test_adapter.py      |  3 +++
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/providers/src/airflow/providers/openlineage/plugins/adapter.py b/providers/src/airflow/providers/openlineage/plugins/adapter.py
index 51eb649adb..daa154693a 100644
--- a/providers/src/airflow/providers/openlineage/plugins/adapter.py
+++ b/providers/src/airflow/providers/openlineage/plugins/adapter.py
@@ -32,7 +32,6 @@ from openlineage.client.facet_v2 import (
     nominal_time_run,
     ownership_job,
     parent_run,
-    processing_engine_run,
     source_code_location_job,
 )
 from openlineage.client.uuid import generate_static_uuid
@@ -42,6 +41,7 @@ from airflow.providers.openlineage.utils.utils import (
     OpenLineageRedactor,
     get_airflow_debug_facet,
     get_airflow_state_run_facet,
+    get_processing_engine_facet,
 )
 from airflow.stats import Stats
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -197,18 +197,10 @@ class OpenLineageAdapter(LoggingMixin):
         :param task: metadata container with information extracted from operator
         :param run_facets: custom run facets
         """
-        from airflow.version import version as AIRFLOW_VERSION
-
-        processing_engine_version_facet = processing_engine_run.ProcessingEngineRunFacet(
-            version=AIRFLOW_VERSION,
-            name="Airflow",
-            openlineageAdapterVersion=OPENLINEAGE_PROVIDER_VERSION,
-        )
-
         run_facets = run_facets or {}
         if task:
             run_facets = {**task.run_facets, **run_facets}
-        run_facets["processing_engine"] = processing_engine_version_facet  # type: ignore
+        run_facets = {**run_facets, **get_processing_engine_facet()}  # type: ignore
         event = RunEvent(
             eventType=RunState.START,
             eventTime=event_time,
@@ -364,7 +356,7 @@ class OpenLineageAdapter(LoggingMixin):
                     job_name=dag_id,
                     nominal_start_time=nominal_start_time,
                     nominal_end_time=nominal_end_time,
-                    run_facets={**run_facets, **get_airflow_debug_facet()},
+                    run_facets={**run_facets, **get_airflow_debug_facet(), **get_processing_engine_facet()},
                 ),
                 inputs=[],
                 outputs=[],
diff --git a/providers/src/airflow/providers/openlineage/utils/utils.py b/providers/src/airflow/providers/openlineage/utils/utils.py
index 3d05d564a7..786c58ff12 100644
--- a/providers/src/airflow/providers/openlineage/utils/utils.py
+++ b/providers/src/airflow/providers/openlineage/utils/utils.py
@@ -38,7 +38,7 @@ from airflow.exceptions import (
 # TODO: move this maybe to Airflow's logic?
 from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
 from airflow.providers.common.compat.assets import Asset
-from airflow.providers.openlineage import conf
+from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
 from airflow.providers.openlineage.plugins.facets import (
     AirflowDagRunFacet,
     AirflowDebugRunFacet,
@@ -65,7 +65,7 @@ from airflow.utils.module_loading import import_string
 
 if TYPE_CHECKING:
     from openlineage.client.event_v2 import Dataset as OpenLineageDataset
-    from openlineage.client.facet_v2 import RunFacet
+    from openlineage.client.facet_v2 import RunFacet, processing_engine_run
 
     from airflow.models import TaskInstance
     from airflow.utils.state import DagRunState, TaskInstanceState
@@ -406,6 +406,18 @@ def _get_all_packages_installed() -> dict[str, str]:
     return {dist.metadata["Name"]: dist.version for dist in metadata.distributions()}
 
 
+def get_processing_engine_facet() -> dict[str, processing_engine_run.ProcessingEngineRunFacet]:
+    from openlineage.client.facet_v2 import processing_engine_run
+
+    return {
+        "processing_engine": processing_engine_run.ProcessingEngineRunFacet(
+            version=AIRFLOW_VERSION,
+            name="Airflow",
+            openlineageAdapterVersion=OPENLINEAGE_PROVIDER_VERSION,
+        )
+    }
+
+
 def get_airflow_debug_facet() -> dict[str, AirflowDebugRunFacet]:
     if not conf.debug_mode():
         return {}
diff --git a/providers/tests/openlineage/plugins/test_adapter.py b/providers/tests/openlineage/plugins/test_adapter.py
index a2af9f393d..154a9b1c29 100644
--- a/providers/tests/openlineage/plugins/test_adapter.py
+++ b/providers/tests/openlineage/plugins/test_adapter.py
@@ -617,6 +617,9 @@ def test_emit_dag_started_event(mock_stats_incr, mock_stats_timer, generate_stat
                         nominalStartTime=event_time.isoformat(),
                         nominalEndTime=event_time.isoformat(),
                     ),
+                    "processing_engine": processing_engine_run.ProcessingEngineRunFacet(
+                        version=ANY, name="Airflow", openlineageAdapterVersion=ANY
+                    ),
                     "airflowDagRun": AirflowDagRunFacet(
                         dag=expected_dag_info,
                         dagRun={