You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ds...@apache.org on 2023/02/12 08:16:22 UTC
[airflow] branch main updated: Add log groomer sidecar to triggerer service (#29392)
This is an automated email from the ASF dual-hosted git repository.
dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e8da520232 Add log groomer sidecar to triggerer service (#29392)
e8da520232 is described below
commit e8da5202327dd7634190d3a3581828490ae5c300
Author: Daniel Standish <15...@users.noreply.github.com>
AuthorDate: Sun Feb 12 00:16:14 2023 -0800
Add log groomer sidecar to triggerer service (#29392)
Instead of copy / pasting the tests I made it an inheritable class.
DRYify the values schema json
---
.../templates/triggerer/triggerer-deployment.yaml | 30 ++++
chart/values.schema.json | 183 ++++++++-------------
chart/values.yaml | 17 ++
.../pre_commit_vendor_k8s_json_schema.py | 5 +-
tests/charts/log_groomer.py | 127 ++++++++++++++
tests/charts/test_scheduler.py | 103 +-----------
tests/charts/test_triggerer.py | 6 +
tests/charts/test_worker.py | 113 +------------
8 files changed, 264 insertions(+), 320 deletions(-)
diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml
index 0d38ac1d66..617d3bc29b 100644
--- a/chart/templates/triggerer/triggerer-deployment.yaml
+++ b/chart/templates/triggerer/triggerer-deployment.yaml
@@ -211,6 +211,36 @@ spec:
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
+ {{- if .Values.triggerer.logGroomerSidecar.enabled }}
+ - name: triggerer-log-groomer
+ resources:
+{{ toYaml .Values.triggerer.logGroomerSidecar.resources | indent 12 }}
+ image: {{ template "airflow_image" . }}
+ imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
+ {{- if .Values.triggerer.logGroomerSidecar.command }}
+ command: {{ tpl (toYaml .Values.triggerer.logGroomerSidecar.command) . | nindent 12 }}
+ {{- end }}
+ {{- if .Values.triggerer.logGroomerSidecar.args }}
+ args: {{ tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }}
+ {{- end }}
+ {{ if .Values.triggerer.logGroomerSidecar.retentionDays }}
+ env:
+ - name: AIRFLOW__LOG_RETENTION_DAYS
+ value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
+ {{- end }}
+ volumeMounts:
+ - name: logs
+ mountPath: {{ template "airflow_logs" . }}
+{{- if .Values.volumeMounts }}
+{{ toYaml .Values.volumeMounts | nindent 12 }}
+{{- end }}
+{{- if .Values.triggerer.extraVolumeMounts }}
+{{ toYaml .Values.triggerer.extraVolumeMounts | indent 12 }}
+{{- end }}
+{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
+{{ include "airflow_webserver_config_mount" . | indent 12 }}
+{{- end }}
+ {{- end }}
{{- if .Values.triggerer.extraContainers }}
{{- toYaml .Values.triggerer.extraContainers | nindent 8 }}
{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 803dadda77..84aa3e26e6 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -1583,64 +1583,8 @@
}
},
"logGroomerSidecar": {
- "description": "Configuration for worker log groomer sidecar",
- "type": "object",
- "additionalProperties": false,
- "properties": {
- "enabled": {
- "description": "Whether to deploy the Airflow worker log groomer sidecar.",
- "type": "boolean",
- "default": true
- },
- "command": {
- "description": "Command to use when running the Airflow workers log groomer sidecar (templated).",
- "type": [
- "array",
- "null"
- ],
- "items": {
- "type": "string"
- },
- "default": null
- },
- "args": {
- "description": "Args to use when running the Airflow workers log groomer sidecar (templated).",
- "type": [
- "array",
- "null"
- ],
- "items": {
- "type": "string"
- },
- "default": [
- "bash",
- "/clean-logs"
- ]
- },
- "retentionDays": {
- "description": "Number of days to retain the logs when running the Airflow workers log groomer sidecar.",
- "type": "integer",
- "default": 15
- },
- "resources": {
- "description": "Resources for Airflow workers log groomer sidecar.",
- "type": "object",
- "default": {},
- "examples": [
- {
- "limits": {
- "cpu": "100m",
- "memory": "128Mi"
- },
- "requests": {
- "cpu": "100m",
- "memory": "128Mi"
- }
- }
- ],
- "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
- }
- }
+ "$ref": "#/definitions/logGroomerConfigType",
+ "description": "Configuration for log groomer sidecar"
},
"securityContext": {
"description": "Security context for the worker pod. If not set, the values from `securityContext` will be used.",
@@ -1982,63 +1926,8 @@
}
},
"logGroomerSidecar": {
- "description": "Configuration for the schedulers log groomer sidecar.",
- "type": "object",
- "properties": {
- "enabled": {
- "description": "Whether to deploy the Airflow scheduler log groomer sidecar.",
- "type": "boolean",
- "default": true
- },
- "command": {
- "description": "Command to use when running the Airflow scheduler log groomer sidecar (templated).",
- "type": [
- "array",
- "null"
- ],
- "items": {
- "type": "string"
- },
- "default": null
- },
- "args": {
- "description": "Args to use when running the Airflow scheduler log groomer sidecar (templated).",
- "type": [
- "array",
- "null"
- ],
- "items": {
- "type": "string"
- },
- "default": [
- "bash",
- "/clean-logs"
- ]
- },
- "retentionDays": {
- "description": "Number of days to retain the logs when running the Airflow scheduler log groomer sidecar.",
- "type": "integer",
- "default": 15
- },
- "resources": {
- "description": "Resources for log groomer sidecar.",
- "type": "object",
- "default": {},
- "examples": [
- {
- "limits": {
- "cpu": "100m",
- "memory": "128Mi"
- },
- "requests": {
- "cpu": "100m",
- "memory": "128Mi"
- }
- }
- ],
- "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
- }
- }
+ "$ref": "#/definitions/logGroomerConfigType",
+ "description": "Configuration for log groomer sidecar"
},
"securityContext": {
"description": "Security context for the scheduler pod. If not set, the values from `securityContext` will be used.",
@@ -2424,6 +2313,10 @@
}
]
},
+ "logGroomerSidecar": {
+ "$ref": "#/definitions/logGroomerConfigType",
+ "description": "Configuration for log groomer sidecar"
+ },
"waitForMigrations": {
"description": "wait-for-airflow-migrations init container.",
"type": "object",
@@ -8319,6 +8212,66 @@
"type": "integer"
}
]
+ },
+ "logGroomerConfigType": {
+ "description": "Configuration for log groomer sidecar",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "enabled": {
+ "description": "Whether to deploy the Airflow log groomer sidecar.",
+ "type": "boolean",
+ "default": true
+ },
+ "command": {
+ "description": "Command to use when running the Airflow log groomer sidecar (templated).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": null
+ },
+ "args": {
+ "description": "Args to use when running the Airflow log groomer sidecar (templated).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": [
+ "bash",
+ "/clean-logs"
+ ]
+ },
+ "retentionDays": {
+ "description": "Number of days to retain the logs when running the Airflow log groomer sidecar.",
+ "type": "integer",
+ "default": 15
+ },
+ "resources": {
+ "description": "Resources for Airflow log groomer sidecar.",
+ "type": "object",
+ "default": {},
+ "examples": [
+ {
+ "limits": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ },
+ "requests": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ }
+ }
+ ],
+ "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+ }
+ }
}
}
}
diff --git a/chart/values.yaml b/chart/values.yaml
index 60a9575917..ff5ddec1e4 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -1165,6 +1165,23 @@ triggerer:
# Labels specific to triggerer objects and pods
labels: {}
+ logGroomerSidecar:
+ # Whether to deploy the Airflow triggerer log groomer sidecar.
+ enabled: true
+ # Command to use when running the Airflow triggerer log groomer sidecar (templated).
+ command: ~
+ # Args to use when running the Airflow triggerer log groomer sidecar (templated).
+ args: ["bash", "/clean-logs"]
+ # Number of days to retain logs
+ retentionDays: 15
+ resources: {}
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
+
waitForMigrations:
# Whether to create init container to wait for db migrations
enabled: true
diff --git a/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py b/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
index b4ecff3581..b4ed4b4508 100755
--- a/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
+++ b/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
@@ -68,7 +68,10 @@ while True:
starting_refs = refs
for ref in refs:
ref_id = ref.split("/")[-1]
- schema["definitions"][ref_id] = defs["definitions"][ref_id]
+ remote_def = defs["definitions"].get(ref_id)
+ if not remote_def:
+ continue
+ schema["definitions"][ref_id] = remote_def
refs = set(find_refs(schema["definitions"]))
if refs == starting_refs:
break
diff --git a/tests/charts/log_groomer.py b/tests/charts/log_groomer.py
new file mode 100644
index 0000000000..5a0e2310aa
--- /dev/null
+++ b/tests/charts/log_groomer.py
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+import pytest
+
+from tests.charts.helm_template_generator import render_chart
+
+
+class LogGroomerTestBase:
+ obj_name: str = ""
+ folder: str = ""
+
+ def test_log_groomer_collector_default_enabled(self):
+ docs = render_chart(show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"])
+ assert 2 == len(jmespath.search("spec.template.spec.containers", docs[0]))
+ assert f"{self.obj_name}-log-groomer" in [
+ c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])
+ ]
+
+ def test_log_groomer_collector_can_be_disabled(self):
+ docs = render_chart(
+ values={f"{self.folder}": {"logGroomerSidecar": {"enabled": False}}},
+ show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
+ )
+ actual = jmespath.search("spec.template.spec.containers", docs[0])
+
+ assert len(actual) == 1
+
+ def test_log_groomer_collector_default_command_and_args(self):
+ docs = render_chart(show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"])
+
+ assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None
+ assert ["bash", "/clean-logs"] == jmespath.search("spec.template.spec.containers[1].args", docs[0])
+
+ def test_log_groomer_collector_default_retention_days(self):
+ docs = render_chart(show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"])
+
+ assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
+ "spec.template.spec.containers[1].env[0].name", docs[0]
+ )
+ assert "15" == jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0])
+
+ @pytest.mark.parametrize("command", [None, ["custom", "command"]])
+ @pytest.mark.parametrize("args", [None, ["custom", "args"]])
+ def test_log_groomer_command_and_args_overrides(self, command, args):
+ docs = render_chart(
+ values={f"{self.folder}": {"logGroomerSidecar": {"command": command, "args": args}}},
+ show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
+ )
+
+ assert command == jmespath.search("spec.template.spec.containers[1].command", docs[0])
+ assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0])
+
+ def test_log_groomer_command_and_args_overrides_are_templated(self):
+ docs = render_chart(
+ values={
+ f"{self.folder}": {
+ "logGroomerSidecar": {
+ "command": ["{{ .Release.Name }}"],
+ "args": ["{{ .Release.Service }}"],
+ }
+ }
+ },
+ show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
+ )
+
+ assert ["release-name"] == jmespath.search("spec.template.spec.containers[1].command", docs[0])
+ assert ["Helm"] == jmespath.search("spec.template.spec.containers[1].args", docs[0])
+
+ @pytest.mark.parametrize("retention_days, retention_result", [(None, None), (30, "30")])
+ def test_log_groomer_retention_days_overrides(self, retention_days, retention_result):
+ docs = render_chart(
+ values={f"{self.folder}": {"logGroomerSidecar": {"retentionDays": retention_days}}},
+ show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
+ )
+
+ if retention_result:
+ assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
+ "spec.template.spec.containers[1].env[0].name", docs[0]
+ )
+ assert retention_result == jmespath.search(
+ "spec.template.spec.containers[1].env[0].value", docs[0]
+ )
+ else:
+ assert jmespath.search("spec.template.spec.containers[1].env", docs[0]) is None
+
+ def test_log_groomer_resources(self):
+ docs = render_chart(
+ values={
+ f"{self.folder}": {
+ "logGroomerSidecar": {
+ "resources": {
+ "requests": {"memory": "2Gi", "cpu": "1"},
+ "limits": {"memory": "3Gi", "cpu": "2"},
+ }
+ }
+ }
+ },
+ show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
+ )
+
+ assert {
+ "limits": {
+ "cpu": "2",
+ "memory": "3Gi",
+ },
+ "requests": {
+ "cpu": "1",
+ "memory": "2Gi",
+ },
+ } == jmespath.search("spec.template.spec.containers[1].resources", docs[0])
diff --git a/tests/charts/test_scheduler.py b/tests/charts/test_scheduler.py
index bf53ff91a9..1ae14e4048 100644
--- a/tests/charts/test_scheduler.py
+++ b/tests/charts/test_scheduler.py
@@ -20,6 +20,7 @@ import jmespath
import pytest
from tests.charts.helm_template_generator import render_chart
+from tests.charts.log_groomer import LogGroomerTestBase
class TestScheduler:
@@ -517,77 +518,6 @@ class TestScheduler:
assert ["release-name"] == jmespath.search("spec.template.spec.containers[0].command", docs[0])
assert ["Helm"] == jmespath.search("spec.template.spec.containers[0].args", docs[0])
- def test_log_groomer_collector_can_be_disabled(self):
- docs = render_chart(
- values={"scheduler": {"logGroomerSidecar": {"enabled": False}}},
- show_only=["templates/scheduler/scheduler-deployment.yaml"],
- )
- assert 1 == len(jmespath.search("spec.template.spec.containers", docs[0]))
-
- def test_log_groomer_collector_default_command_and_args(self):
- docs = render_chart(show_only=["templates/scheduler/scheduler-deployment.yaml"])
-
- assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None
- assert ["bash", "/clean-logs"] == jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- def test_log_groomer_collector_default_retention_days(self):
- docs = render_chart(show_only=["templates/scheduler/scheduler-deployment.yaml"])
-
- assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
- "spec.template.spec.containers[1].env[0].name", docs[0]
- )
- assert "15" == jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0])
-
- @pytest.mark.parametrize("command", [None, ["custom", "command"]])
- @pytest.mark.parametrize("args", [None, ["custom", "args"]])
- def test_log_groomer_command_and_args_overrides(self, command, args):
- docs = render_chart(
- values={"scheduler": {"logGroomerSidecar": {"command": command, "args": args}}},
- show_only=["templates/scheduler/scheduler-deployment.yaml"],
- )
-
- assert command == jmespath.search("spec.template.spec.containers[1].command", docs[0])
- assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- def test_log_groomer_command_and_args_overrides_are_templated(self):
- docs = render_chart(
- values={
- "scheduler": {
- "logGroomerSidecar": {
- "command": ["{{ .Release.Name }}"],
- "args": ["{{ .Release.Service }}"],
- }
- }
- },
- show_only=["templates/scheduler/scheduler-deployment.yaml"],
- )
-
- assert ["release-name"] == jmespath.search("spec.template.spec.containers[1].command", docs[0])
- assert ["Helm"] == jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- @pytest.mark.parametrize(
- "retention_days, retention_result",
- [
- (None, None),
- (30, "30"),
- ],
- )
- def test_log_groomer_retention_days_overrides(self, retention_days, retention_result):
- docs = render_chart(
- values={"scheduler": {"logGroomerSidecar": {"retentionDays": retention_days}}},
- show_only=["templates/scheduler/scheduler-deployment.yaml"],
- )
-
- if retention_result:
- assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
- "spec.template.spec.containers[1].env[0].name", docs[0]
- )
- assert retention_result == jmespath.search(
- "spec.template.spec.containers[1].env[0].value", docs[0]
- )
- else:
- assert jmespath.search("spec.template.spec.containers[1].env", docs[0]) is None
-
@pytest.mark.parametrize(
"dags_values",
[
@@ -655,32 +585,6 @@ class TestScheduler:
c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0])
]
- def test_log_groomer_resources(self):
- docs = render_chart(
- values={
- "scheduler": {
- "logGroomerSidecar": {
- "resources": {
- "requests": {"memory": "2Gi", "cpu": "1"},
- "limits": {"memory": "3Gi", "cpu": "2"},
- }
- }
- }
- },
- show_only=["templates/scheduler/scheduler-deployment.yaml"],
- )
-
- assert {
- "limits": {
- "cpu": "2",
- "memory": "3Gi",
- },
- "requests": {
- "cpu": "1",
- "memory": "2Gi",
- },
- } == jmespath.search("spec.template.spec.containers[1].resources", docs[0])
-
def test_persistence_volume_annotations(self):
docs = render_chart(
values={"executor": "LocalExecutor", "workers": {"persistence": {"annotations": {"foo": "bar"}}}},
@@ -736,6 +640,11 @@ class TestSchedulerNetworkPolicy:
assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value"
+class TestSchedulerLogGroomer(LogGroomerTestBase):
+ obj_name = "scheduler"
+ folder = "scheduler"
+
+
class TestSchedulerService:
@pytest.mark.parametrize(
"executor, creates_service",
diff --git a/tests/charts/test_triggerer.py b/tests/charts/test_triggerer.py
index 7beb872dca..db08c56160 100644
--- a/tests/charts/test_triggerer.py
+++ b/tests/charts/test_triggerer.py
@@ -20,6 +20,7 @@ import jmespath
import pytest
from tests.charts.helm_template_generator import render_chart
+from tests.charts.log_groomer import LogGroomerTestBase
class TestTriggerer:
@@ -561,3 +562,8 @@ class TestTriggererServiceAccount:
assert "test_label" in jmespath.search("metadata.labels", docs[0])
assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value"
+
+
+class TestTriggererLogGroomer(LogGroomerTestBase):
+ obj_name = "triggerer"
+ folder = "triggerer"
diff --git a/tests/charts/test_worker.py b/tests/charts/test_worker.py
index 76a511a389..ef30559b70 100644
--- a/tests/charts/test_worker.py
+++ b/tests/charts/test_worker.py
@@ -20,6 +20,7 @@ import jmespath
import pytest
from tests.charts.helm_template_generator import render_chart
+from tests.charts.log_groomer import LogGroomerTestBase
class TestWorker:
@@ -541,87 +542,6 @@ class TestWorker:
assert ["release-name"] == jmespath.search("spec.template.spec.containers[0].command", docs[0])
assert ["Helm"] == jmespath.search("spec.template.spec.containers[0].args", docs[0])
- def test_log_groomer_collector_default_enabled(self):
- docs = render_chart(show_only=["templates/workers/worker-deployment.yaml"])
- assert 2 == len(jmespath.search("spec.template.spec.containers", docs[0]))
- assert "worker-log-groomer" in [
- c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])
- ]
-
- def test_log_groomer_collector_can_be_disabled(self):
- docs = render_chart(
- values={"workers": {"logGroomerSidecar": {"enabled": False}}},
- show_only=["templates/workers/worker-deployment.yaml"],
- )
- assert 1 == len(jmespath.search("spec.template.spec.containers", docs[0]))
- assert "worker-log-groomer" not in [
- c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])
- ]
-
- def test_log_groomer_default_command_and_args(self):
- docs = render_chart(show_only=["templates/workers/worker-deployment.yaml"])
-
- assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None
- assert ["bash", "/clean-logs"] == jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- def test_log_groomer_collector_default_retention_days(self):
- docs = render_chart(show_only=["templates/workers/worker-deployment.yaml"])
-
- assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
- "spec.template.spec.containers[1].env[0].name", docs[0]
- )
- assert "15" == jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0])
-
- @pytest.mark.parametrize("command", [None, ["custom", "command"]])
- @pytest.mark.parametrize("args", [None, ["custom", "args"]])
- def test_log_groomer_command_and_args_overrides(self, command, args):
- docs = render_chart(
- values={"workers": {"logGroomerSidecar": {"command": command, "args": args}}},
- show_only=["templates/workers/worker-deployment.yaml"],
- )
-
- assert command == jmespath.search("spec.template.spec.containers[1].command", docs[0])
- assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- def test_log_groomer_command_and_args_overrides_are_templated(self):
- docs = render_chart(
- values={
- "workers": {
- "logGroomerSidecar": {
- "command": ["{{ .Release.Name }}"],
- "args": ["{{ .Release.Service }}"],
- }
- }
- },
- show_only=["templates/workers/worker-deployment.yaml"],
- )
-
- assert ["release-name"] == jmespath.search("spec.template.spec.containers[1].command", docs[0])
- assert ["Helm"] == jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- @pytest.mark.parametrize(
- "retention_days, retention_result",
- [
- (None, None),
- (30, "30"),
- ],
- )
- def test_log_groomer_retention_days_overrides(self, retention_days, retention_result):
- docs = render_chart(
- values={"workers": {"logGroomerSidecar": {"retentionDays": retention_days}}},
- show_only=["templates/workers/worker-deployment.yaml"],
- )
-
- if retention_result:
- assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
- "spec.template.spec.containers[1].env[0].name", docs[0]
- )
- assert retention_result == jmespath.search(
- "spec.template.spec.containers[1].env[0].value", docs[0]
- )
- else:
- assert jmespath.search("spec.template.spec.containers[1].env", docs[0]) is None
-
def test_dags_gitsync_sidecar_and_init_container(self):
docs = render_chart(
values={"dags": {"gitSync": {"enabled": True}}},
@@ -647,32 +567,6 @@ class TestWorker:
c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0])
]
- def test_log_groomer_resources(self):
- docs = render_chart(
- values={
- "workers": {
- "logGroomerSidecar": {
- "resources": {
- "requests": {"memory": "2Gi", "cpu": "1"},
- "limits": {"memory": "3Gi", "cpu": "2"},
- }
- }
- }
- },
- show_only=["templates/workers/worker-deployment.yaml"],
- )
-
- assert {
- "limits": {
- "cpu": "2",
- "memory": "3Gi",
- },
- "requests": {
- "cpu": "1",
- "memory": "2Gi",
- },
- } == jmespath.search("spec.template.spec.containers[1].resources", docs[0])
-
def test_persistence_volume_annotations(self):
docs = render_chart(
values={"workers": {"persistence": {"annotations": {"foo": "bar"}}}},
@@ -693,6 +587,11 @@ class TestWorker:
assert jmespath.search("metadata.annotations", docs[0])["test_annotation"] == "test_annotation_value"
+class TestWorkerLogGroomer(LogGroomerTestBase):
+ obj_name = "worker"
+ folder = "workers"
+
+
class TestWorkerKedaAutoScaler:
def test_should_add_component_specific_labels(self):
docs = render_chart(