You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by el...@apache.org on 2021/09/30 14:54:15 UTC
[airflow] branch main updated: Add default weight rule
configuration option (#18627)
This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d79f506 Add default weight rule configuration option (#18627)
d79f506 is described below
commit d79f506213297dc0dc034d6df3226361b6f95d7a
Author: eladkal <45...@users.noreply.github.com>
AuthorDate: Thu Sep 30 17:53:55 2021 +0300
Add default weight rule configuration option (#18627)
* Add default weight rule configuration option
---
airflow/config_templates/config.yml | 7 +++++++
airflow/config_templates/default_airflow.cfg | 3 +++
airflow/configuration.py | 15 +++++++++++++++
airflow/models/baseoperator.py | 2 +-
tests/core/test_configuration.py | 9 +++++++++
tests/models/test_baseoperator.py | 9 +++++++++
6 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index 42d9bca..4c7cfa8 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -354,6 +354,13 @@
type: string
example: ~
default: "0"
+ - name: default_task_weight_rule
+ description: |
+ The weighting method used for the effective total priority weight of the task
+ version_added: 2.2.0
+ type: string
+ example: ~
+ default: "downstream"
- name: min_serialized_dag_update_interval
description: |
Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index c7ede49..464d5f1 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -207,6 +207,9 @@ dag_discovery_safe_mode = True
# The number of retries each task is going to have by default. Can be overridden at dag or task level.
default_task_retries = 0
+# The weighting method used for the effective total priority weight of the task
+default_task_weight_rule = downstream
+
# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.
min_serialized_dag_update_interval = 30
diff --git a/airflow/configuration.py b/airflow/configuration.py
index fb2078a..b139887 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -38,6 +38,7 @@ from airflow.exceptions import AirflowConfigException
from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend
from airflow.utils import yaml
from airflow.utils.module_loading import import_string
+from airflow.utils.weight_rule import WeightRule
log = logging.getLogger(__name__)
@@ -193,6 +194,8 @@ class AirflowConfigParser(ConfigParser):
},
}
+ enums = {("core", "default_task_weight_rule"): WeightRule}
+
# This method transforms option names on every read, get, or set operation.
# This changes from the default behaviour of ConfigParser from lowercasing
# to instead be case-preserving
@@ -212,6 +215,8 @@ class AirflowConfigParser(ConfigParser):
self._validate_config_dependencies()
+ self._validate_enums()
+
for section, replacement in self.deprecated_values.items():
for name, info in replacement.items():
old, new, version = info
@@ -229,6 +234,16 @@ class AirflowConfigParser(ConfigParser):
self.is_validated = True
+ def _validate_enums(self):
+ """Validate that enum type config has an accepted value"""
+ for (section, setting), enum_class in self.enums.items():
+ if self.has_option(section, setting):
+ value = self.get(section, setting)
+ if not enum_class.is_valid(value):
+ raise AirflowConfigException(
+ f"{value} is not an accepted config for [{section}] {setting}"
+ )
+
def _validate_config_dependencies(self):
"""
Validate that config values aren't invalid given other config values
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index 800881f..58b9e46 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -497,7 +497,7 @@ class BaseOperator(Operator, LoggingMixin, TaskMixin, metaclass=BaseOperatorMeta
params: Optional[Dict] = None,
default_args: Optional[Dict] = None,
priority_weight: int = 1,
- weight_rule: str = WeightRule.DOWNSTREAM,
+ weight_rule: str = conf.get('core', 'default_task_weight_rule', fallback=WeightRule.DOWNSTREAM),
queue: str = conf.get('operators', 'default_queue'),
pool: Optional[str] = None,
pool_slots: int = 1,
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index d651de1..8c8e6d7 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -696,3 +696,12 @@ notacommand = OK
def test_confirm_unittest_mod(self):
assert conf.get('core', 'unit_test_mode')
+
+ def test_enum_default_task_weight_rule_from_conf(self):
+ test_conf = AirflowConfigParser(default_config='')
+ test_conf.read_dict({'core': {'default_task_weight_rule': 'sidestream'}})
+ with pytest.raises(AirflowConfigException) as ctx:
+ test_conf.validate()
+ exception = str(ctx.value)
+ message = "sidestream is not an accepted config for [core] default_task_weight_rule"
+ assert message == exception
diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py
index a6c1ee5..f8fb990 100644
--- a/tests/models/test_baseoperator.py
+++ b/tests/models/test_baseoperator.py
@@ -30,6 +30,7 @@ from airflow.models.baseoperator import BaseOperator, BaseOperatorMeta, chain, c
from airflow.utils.edgemodifier import Label
from airflow.utils.task_group import TaskGroup
from airflow.utils.trigger_rule import TriggerRule
+from airflow.utils.weight_rule import WeightRule
from tests.models import DEFAULT_DATE
from tests.test_utils.mock_operators import DeprecatedOperator, MockNamedTuple, MockOperator
@@ -565,6 +566,14 @@ class TestBaseOperator:
assert op1.trigger_rule == TriggerRule.ALWAYS
+ def test_weight_rule_default(self):
+ op = BaseOperator(task_id="test_task")
+ assert WeightRule.DOWNSTREAM == op.weight_rule
+
+ def test_weight_rule_override(self):
+ op = BaseOperator(task_id="test_task", weight_rule="upstream")
+ assert WeightRule.UPSTREAM == op.weight_rule
+
def test_init_subclass_args():
class InitSubclassOp(BaseOperator):