You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "dingo4dev (via GitHub)" <gi...@apache.org> on 2026/07/04 05:27:25 UTC

Re: [PR] Fix asset event scheduling race condition and consistency [airflow]

dingo4dev commented on code in PR #62501:
URL: https://github.com/apache/airflow/pull/62501#discussion_r3522630669


##########
airflow-core/src/airflow/assets/manager.py:
##########
@@ -789,33 +819,46 @@ def _get_or_create_apdr(
             return apdr
 
     @classmethod
-    def _queue_dagruns_nonpartitioned_slow_path(
-        cls, asset_id: int, dags_to_queue: set[DagModel], session: Session
+    def _queue_dagruns_nonpartitioned_mysql(
+        cls, asset_id: int, dags_to_queue: set[DagModel], event: AssetEvent, session: Session
     ) -> None:
-        def _queue_dagrun_if_needed(dag: DagModel) -> str | None:
-            item = AssetDagRunQueue(target_dag_id=dag.dag_id, asset_id=asset_id)
-            # Don't error whole transaction when a single RunQueue item conflicts.
-            # https://docs.sqlalchemy.org/en/14/orm/session_transaction.html#using-savepoint
-            try:
-                with session.begin_nested():
-                    session.merge(item)
-            except exc.IntegrityError:
-                cls.logger().debug("Skipping record %s", item, exc_info=True)
-            return dag.dag_id
+        from sqlalchemy import case
+        from sqlalchemy.dialects.mysql import insert
 
-        queued_results = (_queue_dagrun_if_needed(dag) for dag in dags_to_queue)
-        if queued_dag_ids := [r for r in queued_results if r is not None]:
-            cls.logger().debug("consuming dag ids %s", queued_dag_ids)
+        values = [{"target_dag_id": dag.dag_id} for dag in dags_to_queue]
+        stmt = insert(AssetDagRunQueue).values(asset_id=asset_id, created_at=event.timestamp)
+
+        update_stmt = stmt.on_duplicate_key_update(
+            created_at=case(
+                (stmt.inserted.created_at >= AssetDagRunQueue.created_at, stmt.inserted.created_at),
+                else_=AssetDagRunQueue.created_at,
+            )
+        )
+        session.execute(update_stmt, values)
 
     @classmethod
-    def _queue_dagruns_nonpartitioned_postgres(
-        cls, asset_id: int, dags_to_queue: set[DagModel], session: Session
+    def _queue_dagruns_nonpartitioned_conflict_update(
+        cls,
+        asset_id: int,
+        dags_to_queue: set[DagModel],
+        event: AssetEvent,
+        session: Session,
+        dialect_name: str | None,

Review Comment:
   I added this because the statics check (mypy-core-airflow) failed before. 
   maybe this code `dialect_name = inspect(session.get_bind()).dialect.name` may return None



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@airflow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org