You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ra...@apache.org on 2026/02/03 13:25:37 UTC
(airflow) branch backport-61273 created (now e6661ff680a)
This is an automated email from the ASF dual-hosted git repository.
rahulvats pushed a change to branch backport-61273
in repository https://gitbox.apache.org/repos/asf/airflow.git
at e6661ff680a Flatten grid structure endpoint memory consumption (#61273)
This branch includes the following new commits:
new b95447a0329 Merge branch 'v3-1-test' of github.com:apache/airflow into v3-1-test
new e6661ff680a Flatten grid structure endpoint memory consumption (#61273)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow) 02/02: Flatten grid structure endpoint memory consumption (#61273)
Posted by ra...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rahulvats pushed a commit to branch backport-61273
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e6661ff680a24d1fa11519be264a95edf3832755
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Sun Feb 1 07:18:46 2026 -0700
Flatten grid structure endpoint memory consumption (#61273)
The grid structure endpoint was loading all serdags for the shown
dagruns into memory at once, before merging them together.
Now, we load 5 at a time and also expunge so they can be gc'd more
quickly.
---
.../airflow/api_fastapi/core_api/routes/ui/grid.py | 60 ++++++++--------------
1 file changed, 21 insertions(+), 39 deletions(-)
diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
index 0379bd957ca..f0d077b277e 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
@@ -135,6 +135,8 @@ def get_dag_structure(
"""Return dag structure for grid view."""
latest_serdag = _get_latest_serdag(dag_id, session)
latest_dag = latest_serdag.dag
+ latest_serdag_id = latest_serdag.id
+ session.expunge(latest_serdag) # allow GC of serdag; only latest_dag is needed from here
# Retrieve, sort the previous DAG Runs
base_query = select(DagRun.id).where(DagRun.dag_id == dag_id)
@@ -159,12 +161,22 @@ def get_dag_structure(
nodes = [task_group_to_dict_grid(x) for x in task_group_sort(latest_dag.task_group)]
return [GridNodeResponse(**n) for n in nodes]
- serdags = session.scalars(
- select(SerializedDagModel).where(
+ # Process and merge the latest serdag first
+ merged_nodes: list[dict[str, Any]] = []
+ nodes = [task_group_to_dict_grid(x) for x in task_group_sort(latest_dag.task_group)]
+ _merge_node_dicts(merged_nodes, nodes)
+ del latest_dag
+
+ # Process serdags one by one and merge immediately to reduce memory usage.
+ # Use yield_per() for streaming results and expunge each serdag after processing
+ # to allow garbage collection and prevent memory buildup in the session identity map.
+ serdags_query = (
+ select(SerializedDagModel)
+ .where(
# Even though dag_id is filtered in base_query,
# adding this line here can improve the performance of this endpoint
SerializedDagModel.dag_id == dag_id,
- SerializedDagModel.id != latest_serdag.id,
+ SerializedDagModel.id != latest_serdag_id,
SerializedDagModel.dag_version_id.in_(
select(TaskInstance.dag_version_id)
.join(TaskInstance.dag_run)
@@ -174,45 +186,15 @@ def get_dag_structure(
.distinct()
),
)
+ .execution_options(yield_per=5) # balance between peak memory usage and round trips
)
- merged_nodes: list[dict[str, Any]] = []
- dags = [latest_dag]
- for serdag in serdags:
- if serdag:
- dags.append(serdag.dag)
- for dag in dags:
- nodes = [task_group_to_dict_grid(x) for x in task_group_sort(dag.task_group)]
+
+ for serdag in session.scalars(serdags_query):
+ # Merge immediately instead of collecting all DAGs in memory
+ nodes = [task_group_to_dict_grid(x) for x in task_group_sort(serdag.dag.task_group)]
_merge_node_dicts(merged_nodes, nodes)
- # Ensure historical tasks (e.g. removed) that exist in TIs for the selected runs are represented
- def _collect_ids(nodes: list[dict[str, Any]]) -> set[str]:
- ids: set[str] = set()
- for n in nodes:
- nid = n.get("id")
- if nid:
- ids.add(nid)
- children = n.get("children")
- if children:
- ids |= _collect_ids(children) # recurse
- return ids
-
- existing_ids = _collect_ids(merged_nodes)
- historical_task_ids = session.scalars(
- select(TaskInstance.task_id)
- .join(TaskInstance.dag_run)
- .where(TaskInstance.dag_id == dag_id, DagRun.id.in_(run_ids))
- .distinct()
- )
- for task_id in historical_task_ids:
- if task_id not in existing_ids:
- merged_nodes.append(
- {
- "id": task_id,
- "label": task_id,
- "is_mapped": None,
- "children": None,
- }
- )
+ session.expunge(serdag) # to allow garbage collection
return [GridNodeResponse(**n) for n in merged_nodes]
(airflow) 01/02: Merge branch 'v3-1-test' of github.com:apache/airflow into v3-1-test
Posted by ra...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rahulvats pushed a commit to branch backport-61273
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b95447a032916fe53446a1337cf82e371de3eb85
Merge: 0d0c6018b51 82c7b26ae6e
Author: vatsrahul1001 <ra...@gmail.com>
AuthorDate: Tue Feb 3 18:32:48 2026 +0530
Merge branch 'v3-1-test' of github.com:apache/airflow into v3-1-test
.github/CODEOWNERS | 2 +-
.github/actions/breeze/action.yml | 2 +-
.github/actions/install-prek/action.yml | 4 +-
.github/workflows/basic-tests.yml | 10 +-
.github/workflows/release_dockerhub_image.yml | 2 +-
.pre-commit-config.yaml | 9 +-
Dockerfile | 4 +-
Dockerfile.ci | 8 +-
NOTICE | 2 +-
README.md | 12 +-
RELEASE_NOTES.rst | 74 +
airflow-core/NOTICE | 2 +-
.../administration-and-deployment/dag-bundles.rst | 17 +
airflow-core/docs/best-practices.rst | 2 +-
airflow-core/docs/core-concepts/sensors.rst | 64 +
.../docs/installation/supported-versions.rst | 2 +-
.../docs/installation/upgrading_to_airflow3.rst | 7 +-
airflow-core/docs/public-airflow-interface.rst | 56 +-
airflow-core/docs/security/api.rst | 3 +-
airflow-core/docs/templates-ref.rst | 22 +-
airflow-core/docs/tutorial/hitl.rst | 9 +
airflow-core/pyproject.toml | 6 +-
airflow-core/src/airflow/__init__.py | 2 +-
.../src/airflow/api_fastapi/auth/tokens.py | 15 +-
.../src/airflow/api_fastapi/common/parameters.py | 5 +-
.../api_fastapi/core_api/datamodels/pools.py | 6 +-
.../core_api/openapi/v2-rest-api-generated.yaml | 3 +
.../core_api/routes/public/event_logs.py | 9 +-
.../core_api/routes/public/import_error.py | 35 +-
.../airflow/api_fastapi/core_api/routes/ui/dags.py | 1 -
.../src/airflow/api_fastapi/core_api/security.py | 16 +-
airflow-core/src/airflow/assets/manager.py | 9 +-
.../src/airflow/cli/commands/dag_command.py | 18 +-
.../src/airflow/config_templates/config.yml | 9 +-
airflow-core/src/airflow/dag_processing/manager.py | 90 +-
.../src/airflow/dag_processing/processor.py | 28 +-
.../src/airflow/jobs/scheduler_job_runner.py | 23 +-
.../src/airflow/jobs/triggerer_job_runner.py | 8 +-
airflow-core/src/airflow/models/dagbag.py | 34 +
airflow-core/src/airflow/models/pool.py | 17 -
airflow-core/src/airflow/models/serialized_dag.py | 7 +-
airflow-core/src/airflow/models/xcom.py | 10 +-
airflow-core/src/airflow/serialization/helpers.py | 4 +-
.../airflow/serialization/serialized_objects.py | 14 +-
.../src/airflow/ti_deps/deps/trigger_rule_dep.py | 68 +-
.../airflow/ui/openapi-gen/requests/schemas.gen.ts | 5 +-
.../airflow/ui/public/i18n/locales/ca/common.json | 10 +-
.../ui/public/i18n/locales/ca/components.json | 4 +
.../airflow/ui/public/i18n/locales/de/common.json | 10 +-
.../ui/public/i18n/locales/de/components.json | 4 +
.../ui/public/i18n/locales/en/components.json | 2 +-
.../src/airflow/ui/public/i18n/locales/en/dag.json | 6 +
.../airflow/ui/public/i18n/locales/pl/common.json | 12 +-
.../ui/public/i18n/locales/pl/components.json | 4 +
.../ui/public/i18n/locales/zh-TW/common.json | 10 +-
.../ui/public/i18n/locales/zh-TW/components.json | 4 +
.../ui/src/components/DataTable/DataTable.tsx | 2 +
.../airflow/ui/src/layouts/Details/Gantt/utils.ts | 1 +
.../ui/src/layouts/Nav/LanguageSelector.tsx | 37 +-
.../airflow/ui/src/pages/AssetsList/AssetsList.tsx | 11 +-
.../ui/src/pages/Connections/Connections.tsx | 11 +-
airflow-core/src/airflow/ui/src/pages/Dag/Dag.tsx | 7 +
.../src/airflow/ui/src/pages/Dag/DagNotFound.tsx | 70 +
.../pages/Dag/Tasks/TaskFilters/TaskFilters.tsx | 6 +-
airflow-core/src/airflow/ui/src/pages/DagRuns.tsx | 1 +
.../src/airflow/ui/src/pages/DagsList/DagsList.tsx | 17 +-
.../src/airflow/ui/src/pages/ExternalView.tsx | 6 +-
airflow-core/src/airflow/ui/src/pages/Plugins.tsx | 10 +-
.../src/airflow/ui/src/pages/Pools/Pools.tsx | 9 +-
.../src/airflow/ui/src/pages/ReactPlugin.tsx | 24 +-
.../ui/src/pages/TaskInstances/TaskInstances.tsx | 1 +
.../airflow/ui/src/pages/Variables/Variables.tsx | 11 +-
airflow-core/src/airflow/ui/src/theme.ts | 3 -
airflow-core/src/airflow/utils/cli.py | 15 +-
airflow-core/src/airflow/utils/module_loading.py | 23 +-
airflow-core/tests/integration/otel/test_otel.py | 51 +-
.../core_api/routes/public/test_dag_warning.py | 2 +-
.../core_api/routes/public/test_event_logs.py | 4 +-
.../core_api/routes/public/test_import_error.py | 51 +-
.../core_api/routes/public/test_pools.py | 21 +-
.../core_api/routes/public/test_task_instances.py | 10 +-
.../core_api/routes/ui/test_backfills.py | 2 +-
.../core_api/routes/ui/test_structure.py | 10 +-
.../unit/api_fastapi/core_api/test_security.py | 3 +
airflow-core/tests/unit/assets/test_manager.py | 40 +-
.../tests/unit/cli/commands/test_dag_command.py | 9 +-
.../tests/unit/dag_processing/test_manager.py | 178 +-
.../tests/unit/dag_processing/test_processor.py | 29 +
airflow-core/tests/unit/jobs/test_scheduler_job.py | 39 +-
airflow-core/tests/unit/models/test_dag.py | 4 +-
airflow-core/tests/unit/models/test_dagbag.py | 95 +-
airflow-core/tests/unit/models/test_pool.py | 8 -
.../unit/serialization/test_dag_serialization.py | 41 +-
.../tests/unit/utils/test_module_loading.py | 70 +-
.../src/airflowctl/api/datamodels/generated.py | 10 +-
apache_airflow_site.py | 16 -
chart/NOTICE | 2 +-
contributing-docs/05_pull_requests.rst | 23 +
dev/README_RELEASE_AIRFLOW.md | 1 +
dev/README_RELEASE_AIRFLOWCTL.md | 1 +
dev/README_RELEASE_HELM_CHART.md | 92 +-
dev/README_RELEASE_PROVIDERS.md | 1 +
dev/README_RELEASE_PYTHON_CLIENT.md | 9 +-
dev/breeze/doc/03_developer_tasks.rst | 2 +-
dev/breeze/doc/ci/02_images.md | 6 +-
dev/breeze/pyproject.toml | 2 +-
.../src/airflow_breeze/commands/ci_commands.py | 4 +-
.../commands/release_management_commands.py | 16 +-
dev/breeze/src/airflow_breeze/global_constants.py | 6 +-
.../airflow_breeze/utils/check_release_files.py | 11 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 17 +-
dev/breeze/uv.lock | 2008 ++++++++++----------
.../react_plugin_template/README.md | 2 -
.../react_plugin_template/src/global.d.ts | 24 -
.../react_plugin_template/src/main.tsx | 8 +-
.../react_plugin_template/src/theme.ts | 405 +++-
devel-common/pyproject.toml | 2 +-
docker-stack-docs/README.md | 10 +-
.../extending/add-airflow-configuration/Dockerfile | 2 +-
.../extending/add-apt-packages/Dockerfile | 2 +-
.../add-build-essential-extend/Dockerfile | 2 +-
.../extending/add-providers/Dockerfile | 2 +-
.../add-pypi-packages-constraints/Dockerfile | 2 +-
.../extending/add-pypi-packages-uv/Dockerfile | 2 +-
.../extending/add-pypi-packages/Dockerfile | 2 +-
.../extending/add-requirement-packages/Dockerfile | 2 +-
.../extending/custom-providers/Dockerfile | 2 +-
.../extending/embedding-dags/Dockerfile | 2 +-
.../extending/writable-directory/Dockerfile | 2 +-
docker-stack-docs/entrypoint.rst | 14 +-
docs/spelling_wordlist.txt | 2 +
generated/PYPI_README.md | 10 +-
go-sdk/NOTICE | 2 +-
providers/airbyte/NOTICE | 2 +-
providers/alibaba/NOTICE | 2 +-
providers/amazon/NOTICE | 2 +-
providers/apache/beam/NOTICE | 2 +-
providers/apache/cassandra/NOTICE | 2 +-
providers/apache/drill/NOTICE | 2 +-
providers/apache/druid/NOTICE | 2 +-
providers/apache/flink/NOTICE | 2 +-
providers/apache/hdfs/NOTICE | 2 +-
providers/apache/hive/NOTICE | 2 +-
providers/apache/iceberg/NOTICE | 2 +-
providers/apache/impala/NOTICE | 2 +-
providers/apache/kafka/NOTICE | 2 +-
providers/apache/kylin/NOTICE | 2 +-
providers/apache/livy/NOTICE | 2 +-
providers/apache/pig/NOTICE | 2 +-
providers/apache/pinot/NOTICE | 2 +-
providers/apache/spark/NOTICE | 2 +-
providers/apache/tinkerpop/NOTICE | 2 +-
providers/apprise/NOTICE | 2 +-
providers/arangodb/NOTICE | 2 +-
providers/asana/NOTICE | 2 +-
providers/atlassian/jira/NOTICE | 2 +-
providers/celery/NOTICE | 2 +-
providers/cloudant/NOTICE | 2 +-
providers/cncf/kubernetes/NOTICE | 2 +-
providers/cohere/NOTICE | 2 +-
providers/common/compat/NOTICE | 2 +-
providers/common/io/NOTICE | 2 +-
providers/common/messaging/NOTICE | 2 +-
providers/common/sql/NOTICE | 2 +-
providers/databricks/NOTICE | 2 +-
providers/datadog/NOTICE | 2 +-
providers/dbt/cloud/NOTICE | 2 +-
providers/dingding/NOTICE | 2 +-
providers/discord/NOTICE | 2 +-
providers/docker/NOTICE | 2 +-
providers/edge3/NOTICE | 2 +-
providers/elasticsearch/NOTICE | 2 +-
providers/exasol/NOTICE | 2 +-
providers/fab/3rd-party-licenses/LICENSES-ui.txt | 2 +-
providers/fab/NOTICE | 2 +-
providers/facebook/NOTICE | 2 +-
providers/ftp/NOTICE | 2 +-
providers/git/NOTICE | 2 +-
providers/github/NOTICE | 2 +-
providers/google/NOTICE | 2 +-
.../providers/google/3rd-party-licenses/NOTICE | 2 +-
providers/grpc/NOTICE | 2 +-
providers/hashicorp/NOTICE | 2 +-
providers/http/NOTICE | 2 +-
providers/imap/NOTICE | 2 +-
providers/influxdb/NOTICE | 2 +-
providers/jdbc/NOTICE | 2 +-
providers/jenkins/NOTICE | 2 +-
providers/keycloak/NOTICE | 2 +-
providers/microsoft/azure/NOTICE | 2 +-
providers/microsoft/mssql/NOTICE | 2 +-
providers/microsoft/psrp/NOTICE | 2 +-
providers/microsoft/winrm/NOTICE | 2 +-
providers/mongo/NOTICE | 2 +-
providers/mysql/NOTICE | 2 +-
providers/mysql/docs/index.rst | 19 +-
providers/mysql/pyproject.toml | 6 +-
providers/neo4j/NOTICE | 2 +-
providers/odbc/NOTICE | 2 +-
providers/openai/NOTICE | 2 +-
providers/openfaas/NOTICE | 2 +-
providers/openlineage/NOTICE | 2 +-
providers/opensearch/NOTICE | 2 +-
providers/opsgenie/NOTICE | 2 +-
providers/oracle/NOTICE | 2 +-
providers/pagerduty/NOTICE | 2 +-
providers/papermill/NOTICE | 2 +-
providers/pgvector/NOTICE | 2 +-
providers/pinecone/NOTICE | 2 +-
providers/postgres/NOTICE | 2 +-
providers/presto/NOTICE | 2 +-
providers/qdrant/NOTICE | 2 +-
providers/redis/NOTICE | 2 +-
providers/salesforce/NOTICE | 2 +-
providers/samba/NOTICE | 2 +-
providers/segment/NOTICE | 2 +-
providers/sendgrid/NOTICE | 2 +-
providers/sftp/NOTICE | 2 +-
providers/singularity/NOTICE | 2 +-
providers/slack/NOTICE | 2 +-
providers/smtp/NOTICE | 2 +-
providers/snowflake/NOTICE | 2 +-
providers/sqlite/NOTICE | 2 +-
providers/ssh/NOTICE | 2 +-
providers/standard/NOTICE | 2 +-
providers/standard/docs/operators/hitl.rst | 44 +
providers/tableau/NOTICE | 2 +-
providers/telegram/NOTICE | 2 +-
providers/teradata/NOTICE | 2 +-
providers/trino/NOTICE | 2 +-
providers/vertica/NOTICE | 2 +-
providers/weaviate/NOTICE | 2 +-
providers/yandex/NOTICE | 2 +-
providers/ydb/NOTICE | 2 +-
providers/zendesk/NOTICE | 2 +-
pyproject.toml | 8 +-
reproducible_build.yaml | 4 +-
scripts/ci/install_breeze.sh | 2 +-
.../prek/check_contextmanager_class_decorators.py | 0
scripts/ci/prek/check_imports_in_providers.py | 2 +-
...ports_in_providers.py => check_notice_files.py} | 43 +-
scripts/ci/prek/compile_ui_assets_dev.py | 8 +-
scripts/ci/prek/ruff_format.py | 2 +-
scripts/ci/prek/supported_versions.py | 2 +-
scripts/tools/setup_breeze | 2 +-
.../src/airflow_shared/logging/structlog.py | 2 +-
shared/logging/tests/logging/test_structlog.py | 13 +
task-sdk/NOTICE | 2 +-
task-sdk/pyproject.toml | 2 +-
task-sdk/src/airflow/sdk/__init__.py | 2 +-
task-sdk/src/airflow/sdk/definitions/dag.py | 6 +-
.../src/airflow/sdk/execution_time/task_runner.py | 97 +-
.../task_sdk/execution_time/test_task_runner.py | 106 +-
253 files changed, 3352 insertions(+), 1699 deletions(-)