You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@liminal.apache.org by av...@apache.org on 2021/11/23 12:54:29 UTC
[incubator-liminal] branch master updated: [LIMINAL-85]: Liminal delete
This is an automated email from the ASF dual-hosted git repository.
aviemzur pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new 99b33ae [LIMINAL-85]: Liminal delete
99b33ae is described below
commit 99b33aeea1ad5291f9fc6ea31ed11e34b8c59d73
Author: Lidor Ettinger <li...@gmail.com>
AuthorDate: Tue Nov 23 14:54:25 2021 +0200
[LIMINAL-85]: Liminal delete
---
liminal/kubernetes/volume_util.py | 21 +++++++--
scripts/liminal | 18 ++++++++
tests/liminal/kubernetes/__init__.py | 17 +++++++
tests/liminal/kubernetes/test_volume_util.py | 68 ++++++++++++++++++++++++++++
4 files changed, 121 insertions(+), 3 deletions(-)
diff --git a/liminal/kubernetes/volume_util.py b/liminal/kubernetes/volume_util.py
index 835bc0e..49db8b1 100644
--- a/liminal/kubernetes/volume_util.py
+++ b/liminal/kubernetes/volume_util.py
@@ -36,19 +36,26 @@ _LOCAL_VOLUMES = set([])
_kubernetes = client.CoreV1Api()
-def create_local_volumes(liminal_config, base_dir):
+def get_volume_configs(liminal_config, base_dir):
volumes_config = liminal_config.get('volumes', [])
for volume_config in volumes_config:
if 'local' in volume_config:
- logging.info(f'Creating local kubernetes volume if needed: {volume_config}')
path = volume_config['local']['path']
if path.startswith(".."):
path = os.path.join(base_dir, path)
if path.startswith("."):
path = os.path.join(base_dir, path[1:])
volume_config['local']['path'] = path
- create_local_volume(volume_config)
+ return volumes_config
+
+
+def create_local_volumes(liminal_config, base_dir):
+ volumes_config = get_volume_configs(liminal_config, base_dir)
+
+ for volume_config in volumes_config:
+ logging.info(f'Creating local kubernetes volume if needed: {volume_config}')
+ create_local_volume(volume_config)
def create_local_volume(conf, namespace='default') -> None:
@@ -84,6 +91,14 @@ def create_local_volume(conf, namespace='default') -> None:
_LOCAL_VOLUMES.add(name)
+def delete_local_volumes(liminal_config, base_dir):
+ volumes_config = get_volume_configs(liminal_config, base_dir)
+
+ for volume_config in volumes_config:
+ logging.info(f'Delete local kubernetes volume if needed: {volume_config}')
+ delete_local_volume(volume_config['volume'])
+
+
def delete_local_volume(name, namespace='default'):
pvc_name = f'{name}-pvc'
diff --git a/scripts/liminal b/scripts/liminal
index d560ea0..a56643f 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -72,6 +72,7 @@ def build(path):
volume_util.create_local_volumes(config, os.path.dirname(
files_util.resolve_pipeline_source_file(config['name'])))
+
def deploy_liminal_core_internal(liminal_home, clean):
# noinspection PyTypeChecker
with pkg_resources.path(dag, 'liminal_dags.py') as p:
@@ -192,6 +193,23 @@ def start(detached_mode):
else:
docker_compose_command('up', [])
+@cli.command("delete", short_help="delete liminal resource registration")
+@click.option('--path', default=os.getcwd(), help='Delete within this path.')
+@click.option('--clean', is_flag=True, default=False,
+ help="Delete all resource: DAGs, Volumes")
+def delete(path, clean):
+ configs = ConfigUtil(path).safe_load(is_render_variables=True,
+ soft_merge=True)
+ for config in configs:
+ if config.get('volumes'):
+ logging.info(f'local volume is being deleted')
+ volume_util.delete_local_volumes(config, os.path.dirname(
+ files_util.resolve_pipeline_source_file(config['name'])))
+
+ if clean:
+ dir_path = os.path.abspath(environment.get_liminal_home())
+ shutil.rmtree(dir_path)
+ docker_compose_command('down', ['--remove-orphans', '--rmi', 'local'])
def get_docker_compose_paths():
# noinspection PyTypeChecker
diff --git a/tests/liminal/kubernetes/__init__.py b/tests/liminal/kubernetes/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/tests/liminal/kubernetes/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/liminal/kubernetes/test_volume_util.py b/tests/liminal/kubernetes/test_volume_util.py
new file mode 100644
index 0000000..d7a7192
--- /dev/null
+++ b/tests/liminal/kubernetes/test_volume_util.py
@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import unittest
+
+from kubernetes import config
+from liminal.kubernetes import volume_util
+
+try:
+ config.load_kube_config()
+except Exception:
+ msg = "Kubernetes is not running\n"
+ sys.stdout.write(f"INFO: {msg}")
+
+class TestKubernetesVolume(unittest.TestCase):
+ def setUp(self) -> None:
+ self.config = {
+ 'volumes': [
+ {
+ 'volume': 'gettingstartedvol-test',
+ 'claim_name': 'gettingstartedvol-test-pvc',
+ 'local': {
+ 'path': '.'
+ }
+ }
+ ]
+ }
+
+ def test_volume_config(self):
+ volumes_config = volume_util.get_volume_configs(self.config, ".")
+ self.assertEqual(
+ str(self.config['volumes'][0]),
+ str(volumes_config[0]))
+
+ def test_create_volume(self):
+ self._delete_volumes()
+ self._create_volumes()
+ matching_volumes = volume_util._list_persistent_volumes(self.config['volumes'][0]['volume'])
+ self.assertTrue(self.config['volumes'][0]['volume'] in matching_volumes[0]['metadata']['name'],
+ self.config['volumes'][0]['claim_name'] in matching_volumes[0]['spec']['claim_ref']['name'])
+
+ def test_delete_volume(self):
+ self._create_volumes()
+ self._delete_volumes()
+ matching_volumes = volume_util._list_persistent_volumes(self.config['volumes'][0]['volume'])
+ self.assertEqual([],matching_volumes)
+
+ def _create_volumes(self):
+ volume_util.create_local_volumes(self.config, ".")
+
+ def _delete_volumes(self):
+ volume_util.delete_local_volumes(self.config, ".")