You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ds...@apache.org on 2023/10/04 17:29:16 UTC
[airflow] branch main updated: Remove warning about max_tis per query > parallelism (#34742)
This is an automated email from the ASF dual-hosted git repository.
dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 49bb45ca86 Remove warning about max_tis per query > parallelism (#34742)
49bb45ca86 is described below
commit 49bb45ca8662fff9136fc218d9e242bdce1a0afe
Author: Daniel Standish <15...@users.noreply.github.com>
AuthorDate: Wed Oct 4 10:29:04 2023 -0700
Remove warning about max_tis per query > parallelism (#34742)
This warning is erroneous because it's not true in all cases that the max tis per query setting is disregarded when greater than parallelism.
See discussion at https://github.com/apache/airflow/pull/32572#issuecomment-1739787276.
---
airflow/configuration.py | 21 ---------------------
tests/core/test_configuration.py | 16 ----------------
2 files changed, 37 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 0107cb1021..ab26bbbecf 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -717,7 +717,6 @@ class AirflowConfigParser(ConfigParser):
def validate(self):
self._validate_sqlite3_version()
self._validate_enums()
- self._validate_max_tis_per_query()
for section, replacement in self.deprecated_values.items():
for name, info in replacement.items():
@@ -739,26 +738,6 @@ class AirflowConfigParser(ConfigParser):
self._upgrade_postgres_metastore_conn()
self.is_validated = True
- def _validate_max_tis_per_query(self) -> None:
- """
- Check if config ``scheduler.max_tis_per_query`` is not greater than ``core.parallelism``.
-
- If not met, a warning message is printed to guide the user to correct it.
-
- More info: https://github.com/apache/airflow/pull/32572
- """
- max_tis_per_query = self.getint("scheduler", "max_tis_per_query")
- parallelism = self.getint("core", "parallelism")
-
- if max_tis_per_query > parallelism:
- warnings.warn(
- f"Config scheduler.max_tis_per_query (value: {max_tis_per_query}) "
- f"should NOT be greater than core.parallelism (value: {parallelism}). "
- "Will now use core.parallelism as the max task instances per query "
- "instead of specified value.",
- UserWarning,
- )
-
def _upgrade_auth_backends(self):
"""
Ensure a custom auth_backends setting contains session.
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index 2c6ad3ecfa..8137f1031d 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -760,22 +760,6 @@ notacommand = OK
)
assert message == exception
- @mock.patch.dict(
- "os.environ",
- {
- "AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY": "200",
- "AIRFLOW__CORE__PARALLELISM": "100",
- },
- )
- def test_max_tis_per_query_too_high(self):
- test_conf = AirflowConfigParser()
-
- with pytest.warns(UserWarning) as ctx:
- test_conf._validate_max_tis_per_query()
-
- captured_warnings_msg = str(ctx.pop().message)
- assert "max_tis_per_query" in captured_warnings_msg and "core.parallelism" in captured_warnings_msg
-
def test_as_dict_works_without_sensitive_cmds(self):
conf_materialize_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=True)
conf_maintain_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=False)