You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@liminal.apache.org by as...@apache.org on 2022/11/30 13:32:26 UTC
[incubator-liminal] branch master updated: [LIMINAL-105] add k8s secret cfg (#91)
This is an automated email from the ASF dual-hosted git repository.
assafpinhasi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new 3d5de7f [LIMINAL-105] add k8s secret cfg (#91)
3d5de7f is described below
commit 3d5de7f4b58509f72f6791484b9f3280441cfde2
Author: Lidor Ettinger <li...@gmail.com>
AuthorDate: Wed Nov 30 15:32:20 2022 +0200
[LIMINAL-105] add k8s secret cfg (#91)
* add k8s secret cfg
---
docs/liminal/kubernetes/secret_util.md | 76 ++++++++++++++++
liminal/kubernetes/secret_util.py | 111 ++++++++++++++++++++++++
liminal/runners/airflow/executors/kubernetes.py | 11 +++
liminal/runners/airflow/tasks/containerable.py | 1 +
scripts/liminal | 8 +-
tests/runners/airflow/liminal/liminal.yml | 8 ++
tests/runners/airflow/tasks/test_python.py | 12 ++-
tests/test_licenses.py | 2 +-
8 files changed, 224 insertions(+), 5 deletions(-)
diff --git a/docs/liminal/kubernetes/secret_util.md b/docs/liminal/kubernetes/secret_util.md
new file mode 100644
index 0000000..2bb44f3
--- /dev/null
+++ b/docs/liminal/kubernetes/secret_util.md
@@ -0,0 +1,76 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Secret Utils
+
+Each `task` in your pipelines has a reference to an secret from the `secrets` section of your
+liminal.yml file.
+
+```yaml
+---
+name: k8s_secret_example
+secrets:
+ - secret: aws
+ remote_path: "/secrets"
+ local_path_file: "~/.aws/credentials"
+executors:
+ - executor: k8s
+ type: kubernetes
+variables:
+ AWS_CONFIG_FILE: /mnt/credentials
+task_defaults:
+ python:
+ executor: k8s
+ image: amazon/aws-cli:2.7.23
+ executors: 2
+ env_vars:
+ AWS_CONFIG_FILE: "/secrets/credentials"
+ AWS_PROFILE: "default"
+ secrets:
+ - secret: aws
+ mounts:
+ - mount: myaws-creds
+ volume: aws
+ path: /mnt/
+pipelines:
+ - pipeline: k8s_secret_example
+ owner: Bosco Albert Baracus
+ start_date: 1970-01-01
+ timeout_minutes: 10
+ schedule: 0 * 1 * *
+ tasks:
+ - task: my_python_task
+ type: python
+ cmd: aws s3 ls
+```
+
+That example manifest defines a Secret Opaque for AWS credentials used. The values are Base64 strings in the manifest; however, when you use the Secret with a Pod then the kubelet provides the decoded data to the Pod and its containers.
+
+In order to make use of the AWS credentials we define an environment variable `AWS_CONFIG_FILE` to authenticate our requests.
+
+For example, the files generated by the AWS CLI for a default profile configured with aws configure looks similar to the following.
+
+`~/.aws/credentials`
+```
+[default]
+aws_access_key_id=<aws_access_key_id>
+aws_secret_access_key=<aws_secret_access_key>
+region = us-east-1
+aws_session_token=<aws_session_token>
+```
diff --git a/liminal/kubernetes/secret_util.py b/liminal/kubernetes/secret_util.py
new file mode 100644
index 0000000..e5bccc8
--- /dev/null
+++ b/liminal/kubernetes/secret_util.py
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import base64
+import logging
+import os
+import sys
+from pathlib import Path
+from time import sleep
+
+from kubernetes import client, config
+from kubernetes.client import V1Secret
+
+# noinspection PyBroadException
+try:
+ config.load_kube_config()
+except Exception:
+ msg = "Kubernetes is not running\n"
+ sys.stdout.write(f"INFO: {msg}")
+
+_LOG = logging.getLogger('volume_util')
+_LOCAL_VOLUMES = set()
+_kubernetes = client.CoreV1Api()
+
+
+def get_secret_configs(liminal_config):
+ secrets_config = liminal_config.get('secrets', [])
+
+ for volume_config in secrets_config:
+ if 'secret' in volume_config and 'local_path_file' not in volume_config:
+ secret_path = f"{os.getcwd()}/credentials-{volume_config['secret']}.txt"
+ open(secret_path, 'a').close()
+ volume_config['local_path_file'] = secret_path
+ return secrets_config
+
+
+def create_local_secrets(liminal_config):
+ secrets_config = get_secret_configs(liminal_config)
+
+ for secret_config in secrets_config:
+ logging.info(f'Creating local kubernetes secret if needed: {secret_config}')
+ create_secret(secret_config)
+
+
+def create_secret(conf, namespace='default') -> None:
+ name = conf['secret']
+
+ _LOG.info(f'Requested secret {name}')
+
+ if name not in _LOCAL_VOLUMES:
+ _create_secret(namespace, conf, name)
+ sleep(5)
+
+ _LOCAL_VOLUMES.add(name)
+
+
+def _create_secret(namespace, conf, name):
+ _LOG.info(f'Creating kubernetes secret {name} with spec {conf}')
+
+ _kubernetes.create_namespaced_secret(
+ namespace,
+ V1Secret(
+ api_version='v1',
+ kind='Secret',
+ metadata={
+ 'name': name,
+ 'labels': {"apache/incubator-liminal": "liminal.apache.org"},
+ },
+ data={
+ 'credentials': base64.b64encode(
+ Path(os.path.expanduser(conf['local_path_file'])).read_text().encode('ascii')
+ ).decode('ascii')
+ },
+ ),
+ )
+
+
+def delete_local_secrets(liminal_config, base_dir):
+ secrets_config = get_secret_configs(liminal_config, base_dir)
+
+ for secret_config in secrets_config:
+ logging.info(f'Delete local secret if needed: {secret_config}')
+ delete_local_secret(secret_config)
+
+
+def delete_local_secret(name, namespace='default'):
+ matching_secrets = _kubernetes.list_namespaced_secret(namespace, field_selector=f'metadata.name={name}').to_dict()[
+ 'items'
+ ]
+
+ if len(matching_secrets) > 0:
+ _LOG.info(f'Deleting secret {name}')
+ _kubernetes.delete_namespaced_secret(name, namespace)
+
+ if name in _LOCAL_VOLUMES:
+ _LOCAL_VOLUMES.remove(name)
diff --git a/liminal/runners/airflow/executors/kubernetes.py b/liminal/runners/airflow/executors/kubernetes.py
index 14fddb2..409e603 100644
--- a/liminal/runners/airflow/executors/kubernetes.py
+++ b/liminal/runners/airflow/executors/kubernetes.py
@@ -66,6 +66,7 @@ class KubernetesPodExecutor(executor.Executor):
def _volumes(self):
volumes_config = self.liminal_config.get('volumes', [])
+ secrets_config = self.liminal_config.get('secrets', [])
volumes = []
for volume_config in volumes_config:
name = volume_config['volume']
@@ -74,10 +75,20 @@ class KubernetesPodExecutor(executor.Executor):
claim_name = f'{name}-pvc'
volume = V1Volume(name=name, persistent_volume_claim={'claimName': claim_name})
volumes.append(volume)
+
+ for secret_config in secrets_config:
+ name = secret_config['secret']
+ secret = V1Volume(name=name, secret={'secretName': name})
+ volumes.append(secret)
+
return volumes
def __kubernetes_kwargs(self, task: ContainerTask):
config = copy.deepcopy(self.executor_config)
+ for secret in task.secrets:
+ result = next(x for x in self.liminal_config['secrets'] if x['secret'] == secret['secret'])
+ task.mounts.append({'volume': result['secret'], 'path': result['remote_path']})
+
kubernetes_kwargs = {
'task_id': task.task_id,
'image': task.image,
diff --git a/liminal/runners/airflow/tasks/containerable.py b/liminal/runners/airflow/tasks/containerable.py
index 3a70fb3..2904553 100644
--- a/liminal/runners/airflow/tasks/containerable.py
+++ b/liminal/runners/airflow/tasks/containerable.py
@@ -45,6 +45,7 @@ class ContainerTask(task.Task, ABC):
self.env_vars = self.__env_vars(env)
self.image = self.task_config['image']
self.mounts = self.task_config.get('mounts', [])
+ self.secrets = self.task_config.get('secrets', [])
self.cmds, self.arguments = self._kubernetes_cmds_and_arguments()
def _kubernetes_cmds_and_arguments(self):
diff --git a/scripts/liminal b/scripts/liminal
index 01064eb..897796c 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -33,7 +33,7 @@ from liminal.build import liminal_apps_builder
from liminal.core import environment
from liminal.core.config.config import ConfigUtil
from liminal.core.util import files_util
-from liminal.kubernetes import volume_util
+from liminal.kubernetes import volume_util, secret_util
from liminal.logging.logging_setup import logging_initialization
from liminal.runners.airflow import dag
@@ -158,6 +158,12 @@ def deploy_liminal_apps(path, clean):
target_yml_name = os.path.join(environment.get_dags_dir(), relative_path)
pathlib.Path(target_yml_name).parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(config_file, target_yml_name)
+ configs_path = ConfigUtil(os.path.dirname(config_file)).safe_load(is_render_variables=True,
+ soft_merge=True)
+ for config in configs_path:
+ if config.get('secrets'):
+ logging.info(f'Secret volume is being created')
+ secret_util.create_local_secrets(config)
def is_file_empty(file):
try:
diff --git a/tests/runners/airflow/liminal/liminal.yml b/tests/runners/airflow/liminal/liminal.yml
index f6e32c9..a54a7bd 100644
--- a/tests/runners/airflow/liminal/liminal.yml
+++ b/tests/runners/airflow/liminal/liminal.yml
@@ -21,6 +21,10 @@ volumes:
- volume: myvol1
local:
path: /tmp/liminal_tests
+secrets:
+ - secret: aws
+ remote_path: "/mnt"
+ local_path_file: "~/.aws/credentials"
images:
- image: my_python_task_img
type: python
@@ -60,6 +64,10 @@ pipelines:
env_vars:
NUM_FILES: 10
NUM_SPLITS: 3
+ AWS_CONFIG_FILE: "/mnt/credentials"
+ AWS_PROFILE: "dev"
+ secrets:
+ - secret: aws
mounts:
- mount: mymount
volume: myvol1
diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py
index 0195dc6..bb9dd2a 100644
--- a/tests/runners/airflow/tasks/test_python.py
+++ b/tests/runners/airflow/tasks/test_python.py
@@ -22,7 +22,7 @@ import unittest
from unittest import TestCase
from liminal.build import liminal_apps_builder
-from liminal.kubernetes import volume_util
+from liminal.kubernetes import secret_util, volume_util
from liminal.runners.airflow import DummyDag
from liminal.runners.airflow.executors.kubernetes import KubernetesPodExecutor
from liminal.runners.airflow.tasks import python
@@ -31,9 +31,11 @@ from tests.util import dag_test_utils
class TestPythonTask(TestCase):
_VOLUME_NAME = 'myvol1'
+ _SECRET_NAME = 'aws'
def setUp(self) -> None:
volume_util.delete_local_volume(self._VOLUME_NAME)
+ secret_util.delete_local_secret(self._SECRET_NAME)
os.environ['TMPDIR'] = '/tmp'
self.temp_dir = tempfile.mkdtemp()
self.liminal_config = {
@@ -42,9 +44,12 @@ class TestPythonTask(TestCase):
'volume': self._VOLUME_NAME,
'local': {'path': self.temp_dir.replace("/var/folders", "/private/var/folders")},
}
- ]
+ ],
+ 'secrets': [{'secret': self._SECRET_NAME, 'remote_path': "/mnt"}],
}
+
volume_util.create_local_volumes(self.liminal_config, None)
+ secret_util.create_local_secrets(self.liminal_config)
liminal_apps_builder.build_liminal_apps(os.path.join(os.path.dirname(__file__), '../liminal'))
@@ -57,7 +62,7 @@ class TestPythonTask(TestCase):
None,
'my_python_task_img',
'python -u write_inputs.py',
- env_vars={'NUM_FILES': 10, 'NUM_SPLITS': 3},
+ env_vars={'NUM_FILES': 10, 'NUM_SPLITS': 3, 'AWS_CONFIG_FILE': '/mnt/credentials', 'AWS_PROFILE': 'dev'},
)
executor = KubernetesPodExecutor(
task_id='k8s', liminal_config=self.liminal_config, executor_config={'executor': 'k8s', 'name': 'mypod'}
@@ -143,6 +148,7 @@ class TestPythonTask(TestCase):
'image': image,
'env_vars': env_vars if env_vars is not None else {},
'mounts': [{'mount': 'mymount', 'volume': self._VOLUME_NAME, 'path': '/mnt/vol1'}],
+ 'secrets': [{'secret': self._SECRET_NAME}],
}
if executors:
diff --git a/tests/test_licenses.py b/tests/test_licenses.py
index d5c597a..1c2d361 100644
--- a/tests/test_licenses.py
+++ b/tests/test_licenses.py
@@ -34,7 +34,7 @@ EXCLUDED_DIRS = [
'apache_liminal.egg-info',
'.pytest_cache',
]
-EXCLUDED_FILES = ['DISCLAIMER-WIP', 'LICENSE.txt', '.autovenv']
+EXCLUDED_FILES = ['DISCLAIMER-WIP', 'LICENSE.txt', '.autovenv', 'credentials-aws.txt']
PYTHON_LICENSE_HEADER = """
#