You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2022/06/30 17:17:05 UTC
[airflow] branch main updated: Add support for KEDA HPA config to Helm Chart (#24220)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8639c70f18 Add support for KEDA HPA config to Helm Chart (#24220)
8639c70f18 is described below
commit 8639c70f187a7d5b8b4d2f432d2530f6d259eceb
Author: Junyoung Park <sw...@gmail.com>
AuthorDate: Fri Jul 1 02:16:58 2022 +0900
Add support for KEDA HPA config to Helm Chart (#24220)
---
chart/templates/workers/worker-kedaautoscaler.yaml | 4 +
chart/values.schema.json | 85 ++++++++++++++++++++++
chart/values.yaml | 11 +++
tests/charts/test_keda.py | 34 +++++++++
4 files changed, 134 insertions(+)
diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml
index 35965526a8..9fe869facc 100644
--- a/chart/templates/workers/worker-kedaautoscaler.yaml
+++ b/chart/templates/workers/worker-kedaautoscaler.yaml
@@ -41,6 +41,10 @@ spec:
cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }} # Optional. Default: 300 seconds
minReplicaCount: {{ .Values.workers.keda.minReplicaCount }} # Optional. Default: 0
maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }} # Optional. Default: 100
+{{- if .Values.workers.keda.advanced }}
+ advanced:
+{{ toYaml .Values.workers.keda.advanced | indent 4 }}
+{{- end }}
triggers:
- type: postgresql
metadata:
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 1dbfdd2f2e..bd0ac13dcf 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -1249,6 +1249,27 @@
"description": "Maximum number of workers created by KEDA.",
"type": "integer",
"default": 10
+ },
+ "advanced": {
+ "description": "Advanced KEDA configuration.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": false,
+ "properties": {
+ "horizontalPodAutoscalerConfig": {
+ "description": "HorizontalPodAutoscalerConfig specifies horizontal scale config.",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "behavior": {
+ "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.",
+ "type": "object",
+ "default": {},
+ "$ref": "#/definitions/io.k8s.api.autoscaling.v2beta2.HorizontalPodAutoscalerBehavior"
+ }
+ }
+ }
+ }
}
}
},
@@ -4608,6 +4629,70 @@
"type": "object",
"additionalProperties": false
},
+ "io.k8s.api.autoscaling.v2beta2.HPAScalingPolicy": {
+ "description": "HPAScalingPolicy is a single policy which must hold true for a specified past interval.",
+ "properties": {
+ "periodSeconds": {
+ "description": "PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min).",
+ "format": "int32",
+ "type": "integer"
+ },
+ "type": {
+ "description": "Type is used to specify the scaling policy.",
+ "type": "string"
+ },
+ "value": {
+ "description": "Value contains the amount of change which is permitted by the policy. It must be greater than zero",
+ "format": "int32",
+ "type": "integer"
+ }
+ },
+ "required": [
+ "type",
+ "value",
+ "periodSeconds"
+ ],
+ "type": "object",
+ "additionalProperties": false
+ },
+ "io.k8s.api.autoscaling.v2beta2.HPAScalingRules": {
+ "description": "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.",
+ "properties": {
+ "policies": {
+ "description": "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid",
+ "items": {
+ "$ref": "#/definitions/io.k8s.api.autoscaling.v2beta2.HPAScalingPolicy"
+ },
+ "type": "array"
+ },
+ "selectPolicy": {
+ "description": "selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used.",
+ "type": "string"
+ },
+ "stabilizationWindowSeconds": {
+ "description": "StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).",
+ "format": "int32",
+ "type": "integer"
+ }
+ },
+ "type": "object",
+ "additionalProperties": false
+ },
+ "io.k8s.api.autoscaling.v2beta2.HorizontalPodAutoscalerBehavior": {
+ "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively).",
+ "properties": {
+ "scaleDown": {
+ "$ref": "#/definitions/io.k8s.api.autoscaling.v2beta2.HPAScalingRules",
+ "description": "scaleDown is scaling policy for scaling Down. If not set, the default value is to allow to scale down to minReplicas pods, with a 300 second stabilization window (i.e., the highest recommendation for the last 300sec is used)."
+ },
+ "scaleUp": {
+ "$ref": "#/definitions/io.k8s.api.autoscaling.v2beta2.HPAScalingRules",
+ "description": "scaleUp is scaling policy for scaling Up. If not set, the default value is the higher of:\n * increase no more than 4 pods per 60 seconds\n * double the number of pods per 60 seconds\nNo stabilization is used."
+ }
+ },
+ "type": "object",
+ "additionalProperties": false
+ },
"io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource": {
"description": "Represents a Persistent Disk resource in AWS.\n\nAn AWS EBS disk must exist before mounting to a container. The disk must also be in the same AWS zone as the kubelet. An AWS EBS disk can only be mounted as read/write once. AWS EBS volumes support ownership management and SELinux relabeling.",
"properties": {
diff --git a/chart/values.yaml b/chart/values.yaml
index 45d7a58a93..44a40eebac 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -487,6 +487,17 @@ workers:
# Maximum number of workers created by keda
maxReplicaCount: 10
+ # Specify HPA related options
+ advanced: {}
+ # horizontalPodAutoscalerConfig:
+ # behavior:
+ # scaleDown:
+ # stabilizationWindowSeconds: 300
+ # policies:
+ # - type: Percent
+ # value: 100
+ # periodSeconds: 15
+
persistence:
# Enable persistent volumes
enabled: true
diff --git a/tests/charts/test_keda.py b/tests/charts/test_keda.py
index 43acd3996d..e897378a97 100644
--- a/tests/charts/test_keda.py
+++ b/tests/charts/test_keda.py
@@ -52,6 +52,40 @@ class TestKeda:
else:
assert docs == []
+ @parameterized.expand(
+ [
+ ('CeleryExecutor'),
+ ('CeleryKubernetesExecutor'),
+ ]
+ )
+ def test_keda_advanced(self, executor):
+ """
+ Verify keda advanced config.
+ """
+ expected_advanced = {
+ "horizontalPodAutoscalerConfig": {
+ "behavior": {
+ "scaleDown": {
+ "stabilizationWindowSeconds": 300,
+ "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}],
+ }
+ }
+ }
+ }
+ docs = render_chart(
+ values={
+ "workers": {
+ "keda": {
+ "enabled": True,
+ "advanced": expected_advanced,
+ },
+ },
+ "executor": executor,
+ },
+ show_only=["templates/workers/worker-kedaautoscaler.yaml"],
+ )
+ assert jmespath.search("spec.advanced", docs[0]) == expected_advanced
+
@staticmethod
def build_query(executor, concurrency=16, queue=None):
"""Builds the query used by KEDA autoscaler to determine how many workers there should be"""