You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by el...@apache.org on 2021/09/30 14:54:15 UTC

[airflow] branch main updated: Add default weight rule configuration option (#18627)

This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new d79f506  Add default weight rule configuration option (#18627)
d79f506 is described below

commit d79f506213297dc0dc034d6df3226361b6f95d7a
Author: eladkal <45...@users.noreply.github.com>
AuthorDate: Thu Sep 30 17:53:55 2021 +0300

    Add default weight rule configuration option (#18627)
    
    * Add default weight rule configuration option
---
 airflow/config_templates/config.yml          |  7 +++++++
 airflow/config_templates/default_airflow.cfg |  3 +++
 airflow/configuration.py                     | 15 +++++++++++++++
 airflow/models/baseoperator.py               |  2 +-
 tests/core/test_configuration.py             |  9 +++++++++
 tests/models/test_baseoperator.py            |  9 +++++++++
 6 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index 42d9bca..4c7cfa8 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -354,6 +354,13 @@
       type: string
       example: ~
       default: "0"
+    - name: default_task_weight_rule
+      description: |
+        The weighting method used for the effective total priority weight of the task
+      version_added: 2.2.0
+      type: string
+      example: ~
+      default: "downstream"
     - name: min_serialized_dag_update_interval
       description: |
         Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index c7ede49..464d5f1 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -207,6 +207,9 @@ dag_discovery_safe_mode = True
 # The number of retries each task is going to have by default. Can be overridden at dag or task level.
 default_task_retries = 0
 
+# The weighting method used for the effective total priority weight of the task
+default_task_weight_rule = downstream
+
 # Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.
 min_serialized_dag_update_interval = 30
 
diff --git a/airflow/configuration.py b/airflow/configuration.py
index fb2078a..b139887 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -38,6 +38,7 @@ from airflow.exceptions import AirflowConfigException
 from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend
 from airflow.utils import yaml
 from airflow.utils.module_loading import import_string
+from airflow.utils.weight_rule import WeightRule
 
 log = logging.getLogger(__name__)
 
@@ -193,6 +194,8 @@ class AirflowConfigParser(ConfigParser):
         },
     }
 
+    enums = {("core", "default_task_weight_rule"): WeightRule}
+
     # This method transforms option names on every read, get, or set operation.
     # This changes from the default behaviour of ConfigParser from lowercasing
     # to instead be case-preserving
@@ -212,6 +215,8 @@ class AirflowConfigParser(ConfigParser):
 
         self._validate_config_dependencies()
 
+        self._validate_enums()
+
         for section, replacement in self.deprecated_values.items():
             for name, info in replacement.items():
                 old, new, version = info
@@ -229,6 +234,16 @@ class AirflowConfigParser(ConfigParser):
 
         self.is_validated = True
 
+    def _validate_enums(self):
+        """Validate that enum type config has an accepted value"""
+        for (section, setting), enum_class in self.enums.items():
+            if self.has_option(section, setting):
+                value = self.get(section, setting)
+                if not enum_class.is_valid(value):
+                    raise AirflowConfigException(
+                        f"{value} is not an accepted config for [{section}] {setting}"
+                    )
+
     def _validate_config_dependencies(self):
         """
         Validate that config values aren't invalid given other config values
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index 800881f..58b9e46 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -497,7 +497,7 @@ class BaseOperator(Operator, LoggingMixin, TaskMixin, metaclass=BaseOperatorMeta
         params: Optional[Dict] = None,
         default_args: Optional[Dict] = None,
         priority_weight: int = 1,
-        weight_rule: str = WeightRule.DOWNSTREAM,
+        weight_rule: str = conf.get('core', 'default_task_weight_rule', fallback=WeightRule.DOWNSTREAM),
         queue: str = conf.get('operators', 'default_queue'),
         pool: Optional[str] = None,
         pool_slots: int = 1,
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index d651de1..8c8e6d7 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -696,3 +696,12 @@ notacommand = OK
 
     def test_confirm_unittest_mod(self):
         assert conf.get('core', 'unit_test_mode')
+
+    def test_enum_default_task_weight_rule_from_conf(self):
+        test_conf = AirflowConfigParser(default_config='')
+        test_conf.read_dict({'core': {'default_task_weight_rule': 'sidestream'}})
+        with pytest.raises(AirflowConfigException) as ctx:
+            test_conf.validate()
+        exception = str(ctx.value)
+        message = "sidestream is not an accepted config for [core] default_task_weight_rule"
+        assert message == exception
diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py
index a6c1ee5..f8fb990 100644
--- a/tests/models/test_baseoperator.py
+++ b/tests/models/test_baseoperator.py
@@ -30,6 +30,7 @@ from airflow.models.baseoperator import BaseOperator, BaseOperatorMeta, chain, c
 from airflow.utils.edgemodifier import Label
 from airflow.utils.task_group import TaskGroup
 from airflow.utils.trigger_rule import TriggerRule
+from airflow.utils.weight_rule import WeightRule
 from tests.models import DEFAULT_DATE
 from tests.test_utils.mock_operators import DeprecatedOperator, MockNamedTuple, MockOperator
 
@@ -565,6 +566,14 @@ class TestBaseOperator:
 
             assert op1.trigger_rule == TriggerRule.ALWAYS
 
+    def test_weight_rule_default(self):
+        op = BaseOperator(task_id="test_task")
+        assert WeightRule.DOWNSTREAM == op.weight_rule
+
+    def test_weight_rule_override(self):
+        op = BaseOperator(task_id="test_task", weight_rule="upstream")
+        assert WeightRule.UPSTREAM == op.weight_rule
+
 
 def test_init_subclass_args():
     class InitSubclassOp(BaseOperator):