You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/03/09 20:06:59 UTC
[airflow] branch v1-10-stable updated: Fix merging of secrets and
configmaps (#14090)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-stable by this push:
new 7e7657b Fix merging of secrets and configmaps (#14090)
7e7657b is described below
commit 7e7657b073b742c78445d12ac5f47d90bf683361
Author: cris-b <cr...@beebgames.com>
AuthorDate: Tue Mar 9 20:06:42 2021 +0000
Fix merging of secrets and configmaps (#14090)
closes #13023
---
airflow/kubernetes/pod_generator.py | 12 +++--
tests/kubernetes/test_pod_generator.py | 90 ++++++++++++++++++++++++++++++++++
2 files changed, 98 insertions(+), 4 deletions(-)
diff --git a/airflow/kubernetes/pod_generator.py b/airflow/kubernetes/pod_generator.py
index 4df3198..211b027 100644
--- a/airflow/kubernetes/pod_generator.py
+++ b/airflow/kubernetes/pod_generator.py
@@ -506,7 +506,7 @@ class PodGenerator(object):
'volume_mounts',
'mount_path')
client_container = extend_object_field(base_container, client_container, 'env')
- client_container = extend_object_field(base_container, client_container, 'env_from')
+ client_container = extend_object_field(base_container, client_container, 'env_from', None)
client_container = extend_object_field(base_container, client_container, 'ports')
client_container = extend_object_field(base_container, client_container, 'volume_devices')
client_container = merge_objects(base_container, client_container)
@@ -650,10 +650,14 @@ def extend_object_field(base_obj, client_obj, field_name, field_to_merge="name")
setattr(client_obj_cp, field_name, base_obj_field)
return client_obj_cp
- base_obj_set = _get_dict_from_list(base_obj_field, field_to_merge)
- client_obj_set = _get_dict_from_list(client_obj_field, field_to_merge)
+ if field_to_merge is None:
+ # no merge, just append
+ appended_fields = base_obj_field + client_obj_field
+ else:
+ base_obj_set = _get_dict_from_list(base_obj_field, field_to_merge)
+ client_obj_set = _get_dict_from_list(client_obj_field, field_to_merge)
- appended_fields = _merge_list_of_objects(base_obj_set, client_obj_set)
+ appended_fields = _merge_list_of_objects(base_obj_set, client_obj_set)
setattr(client_obj_cp, field_name, appended_fields)
return client_obj_cp
diff --git a/tests/kubernetes/test_pod_generator.py b/tests/kubernetes/test_pod_generator.py
index 1f907f7..8605edd 100644
--- a/tests/kubernetes/test_pod_generator.py
+++ b/tests/kubernetes/test_pod_generator.py
@@ -696,6 +696,96 @@ class TestPodGenerator(unittest.TestCase):
}, sanitized_result)
@mock.patch('uuid.uuid4')
+ def test_construct_pod_env_from_merge_secretref_and_configmapref(self, mock_uuid):
+ mock_uuid.return_value = self.static_uuid
+ executor_config = k8s.V1Pod(
+ spec=k8s.V1PodSpec(
+ containers=[
+ k8s.V1Container(
+ name='',
+ resources=k8s.V1ResourceRequirements(
+ limits={
+ 'cpu': '1m',
+ 'memory': '1G'
+ }
+ ),
+ env_from=[k8s.V1EnvFromSource(
+ config_map_ref=k8s.V1ConfigMapEnvSource("test_configmap"))
+ ]
+ )
+ ]
+ )
+ )
+ worker_config = k8s.V1Pod(
+ spec=k8s.V1PodSpec(
+ containers=[
+ k8s.V1Container(
+ name='',
+ resources=k8s.V1ResourceRequirements(
+ limits={
+ 'cpu': '1m',
+ 'memory': '1G'
+ }
+ ),
+ env_from=[
+ k8s.V1EnvFromSource(secret_ref=k8s.V1SecretEnvSource("test_secretref"))
+ ]
+ )
+ ]
+ )
+
+ )
+
+ result = PodGenerator.construct_pod(
+ 'dag_id',
+ 'task_id',
+ 'pod_id',
+ self.try_number,
+ "kube_image",
+ self.execution_date,
+ ['command'],
+ executor_config,
+ worker_config,
+ 'namespace',
+ 'uuid',
+ )
+ sanitized_result = self.k8s_client.sanitize_for_serialization(result)
+
+ self.assertEqual({
+ 'apiVersion': 'v1',
+ 'kind': 'Pod',
+ 'metadata': self.metadata,
+ 'spec': {
+ 'containers': [{
+ 'args': [],
+ 'command': ['command'],
+ 'env': [],
+ 'envFrom': [
+ {'secretRef': {
+ 'name': 'test_secretref'
+ }},
+ {'configMapRef': {
+ 'name': 'test_configmap'
+ }}
+ ],
+ 'name': 'base',
+ 'image': 'kube_image',
+ 'ports': [],
+ 'resources': {
+ 'limits': {
+ 'cpu': '1m',
+ 'memory': '1G'
+ }
+ },
+ 'volumeMounts': []
+ }],
+ 'hostNetwork': False,
+ 'imagePullSecrets': [],
+ 'volumes': []
+ }
+ }, sanitized_result)
+
+ @mock.patch('uuid.uuid4')
def test_construct_pod_empty_executor_config(self, mock_uuid):
mock_uuid.return_value = self.static_uuid
worker_config = k8s.V1Pod(