You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/07/17 15:46:19 UTC
(airflow) branch main updated: Add gauge metric for 'last_num_of_db_queries' parameter (#40833)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bc93a949dc Add gauge metric for 'last_num_of_db_queries' parameter (#40833)
bc93a949dc is described below
commit bc93a949dcfb05c32dba44832b306d87d202027b
Author: Maksim <ma...@google.com>
AuthorDate: Wed Jul 17 08:46:12 2024 -0700
Add gauge metric for 'last_num_of_db_queries' parameter (#40833)
---
airflow/dag_processing/manager.py | 1 +
.../logging-monitoring/metrics.rst | 69 +++++++++++-----------
2 files changed, 36 insertions(+), 34 deletions(-)
diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py
index 0ebd201b7f..f358dd9ee1 100644
--- a/airflow/dag_processing/manager.py
+++ b/airflow/dag_processing/manager.py
@@ -885,6 +885,7 @@ class DagFileProcessorManager(LoggingMixin):
seconds_ago = (now - last_run).total_seconds()
Stats.gauge(f"dag_processing.last_run.seconds_ago.{file_name}", seconds_ago)
last_num_of_db_queries = self.get_last_num_of_db_queries(file_path)
+ Stats.gauge(f"dag_processing.last_num_of_db_queries.{file_name}", last_num_of_db_queries)
rows.append(
(
diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
index 648e4b4ca0..49ba7cf422 100644
--- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
+++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
@@ -214,40 +214,41 @@ Name Descripti
Gauges
------
-=================================================== ========================================================================
-Name Description
-=================================================== ========================================================================
-``dagbag_size`` Number of DAGs found when the scheduler ran a scan based on its
- configuration
-``dag_processing.import_errors`` Number of errors from trying to parse DAG files
-``dag_processing.total_parse_time`` Seconds taken to scan and import ``dag_processing.file_path_queue_size`` DAG files
-``dag_processing.file_path_queue_size`` Number of DAG files to be considered for the next scan
-``dag_processing.last_run.seconds_ago.<dag_file>`` Seconds since ``<dag_file>`` was last processed
-``scheduler.tasks.starving`` Number of tasks that cannot be scheduled because of no open slot in pool
-``scheduler.tasks.executable`` Number of tasks that are ready for execution (set to queued)
- with respect to pool limits, DAG concurrency, executor state,
- and priority.
-``executor.open_slots`` Number of open slots on executor
-``executor.queued_tasks`` Number of queued tasks on executor
-``executor.running_tasks`` Number of running tasks on executor
-``pool.open_slots.<pool_name>`` Number of open slots in the pool
-``pool.open_slots`` Number of open slots in the pool. Metric with pool_name tagging.
-``pool.queued_slots.<pool_name>`` Number of queued slots in the pool
-``pool.queued_slots`` Number of queued slots in the pool. Metric with pool_name tagging.
-``pool.running_slots.<pool_name>`` Number of running slots in the pool
-``pool.running_slots`` Number of running slots in the pool. Metric with pool_name tagging.
-``pool.deferred_slots.<pool_name>`` Number of deferred slots in the pool
-``pool.deferred_slots`` Number of deferred slots in the pool. Metric with pool_name tagging.
-``pool.scheduled_tasks.<pool_name>`` Number of scheduled tasks in the pool
-``pool.scheduled_tasks`` Number of scheduled tasks in the pool. Metric with pool_name tagging.
-``pool.starving_tasks.<pool_name>`` Number of starving tasks in the pool
-``pool.starving_tasks`` Number of starving tasks in the pool. Metric with pool_name tagging.
-``task.cpu_usage_percent.<dag_id>.<task_id>`` Percentage of CPU used by a task
-``task.mem_usage_percent.<dag_id>.<task_id>`` Percentage of memory used by a task
-``triggers.running.<hostname>`` Number of triggers currently running for a triggerer (described by hostname)
-``triggers.running`` Number of triggers currently running for a triggerer (described by hostname).
- Metric with hostname tagging.
-=================================================== ========================================================================
+==================================================== ========================================================================
+Name Description
+==================================================== ========================================================================
+``dagbag_size`` Number of DAGs found when the scheduler ran a scan based on its
+ configuration
+``dag_processing.import_errors`` Number of errors from trying to parse DAG files
+``dag_processing.total_parse_time`` Seconds taken to scan and import ``dag_processing.file_path_queue_size`` DAG files
+``dag_processing.file_path_queue_size`` Number of DAG files to be considered for the next scan
+``dag_processing.last_run.seconds_ago.<dag_file>`` Seconds since ``<dag_file>`` was last processed
+``dag_processing.last_num_of_db_queries.<dag_file>`` Number of queries to Airflow database during parsing per ``<dag_file>``
+``scheduler.tasks.starving`` Number of tasks that cannot be scheduled because of no open slot in pool
+``scheduler.tasks.executable`` Number of tasks that are ready for execution (set to queued)
+ with respect to pool limits, DAG concurrency, executor state,
+ and priority.
+``executor.open_slots`` Number of open slots on executor
+``executor.queued_tasks`` Number of queued tasks on executor
+``executor.running_tasks`` Number of running tasks on executor
+``pool.open_slots.<pool_name>`` Number of open slots in the pool
+``pool.open_slots`` Number of open slots in the pool. Metric with pool_name tagging.
+``pool.queued_slots.<pool_name>`` Number of queued slots in the pool
+``pool.queued_slots`` Number of queued slots in the pool. Metric with pool_name tagging.
+``pool.running_slots.<pool_name>`` Number of running slots in the pool
+``pool.running_slots`` Number of running slots in the pool. Metric with pool_name tagging.
+``pool.deferred_slots.<pool_name>`` Number of deferred slots in the pool
+``pool.deferred_slots`` Number of deferred slots in the pool. Metric with pool_name tagging.
+``pool.scheduled_tasks.<pool_name>`` Number of scheduled tasks in the pool
+``pool.scheduled_tasks`` Number of scheduled tasks in the pool. Metric with pool_name tagging.
+``pool.starving_tasks.<pool_name>`` Number of starving tasks in the pool
+``pool.starving_tasks`` Number of starving tasks in the pool. Metric with pool_name tagging.
+``task.cpu_usage_percent.<dag_id>.<task_id>`` Percentage of CPU used by a task
+``task.mem_usage_percent.<dag_id>.<task_id>`` Percentage of memory used by a task
+``triggers.running.<hostname>`` Number of triggers currently running for a triggerer (described by hostname)
+``triggers.running`` Number of triggers currently running for a triggerer (described by hostname).
+ Metric with hostname tagging.
+==================================================== ========================================================================
Timers
------