You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@liminal.apache.org by as...@apache.org on 2021/07/27 14:07:01 UTC
[incubator-liminal] branch master updated: [LIMINAL-81] add images
api (#63)
This is an automated email from the ASF dual-hosted git repository.
assafpinhasi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new ee8f5ce [LIMINAL-81] add images api (#63)
ee8f5ce is described below
commit ee8f5ce0eb7988374e470ec4332a7f78f2a0e5b4
Author: Aviem Zur <av...@gmail.com>
AuthorDate: Tue Jul 27 17:06:50 2021 +0300
[LIMINAL-81] add images api (#63)
* [LIMINAL-81] add images api
* Changes after PR review 1
---
.gitignore | 2 +-
README.md | 24 +-
examples/aws-ml-app-demo/liminal.yml | 10 +-
.../aws-ml-app-demo/manifests/aws-ml-app-demo.yaml | 18 +
examples/liminal-getting-started/liminal.yml | 21 +-
.../{service => image}/python_server/Dockerfile | 0
.../{service => image}/python_server/__init__.py | 0
.../python_server/liminal_python_server.py | 0
.../python_server/python_server.py | 3 +-
.../python_server/python_server_requirements.txt | 0
liminal/build/image_builder.py | 4 -
liminal/build/liminal_apps_builder.py | 72 +--
liminal/build/service/__init__.py | 17 -
liminal/build/service/service_image_builder.py | 18 -
liminal/core/config/config.py | 33 +-
liminal/kubernetes/volume_util.py | 52 +-
tests/liminal/core/config/test_config.py | 543 +++++++++++----------
.../python/test_python_server_image_builder.py | 46 +-
tests/runners/airflow/liminal/liminal.yml | 28 +-
tests/runners/apps/test_app/extra/liminal.yml | 2 +
tests/runners/apps/test_app/liminal.yml | 8 +-
21 files changed, 460 insertions(+), 441 deletions(-)
diff --git a/.gitignore b/.gitignore
index aba3d5b..fcf8bcd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,7 @@ venv
*.pyc
pip-selfcheck.json
.DS_Store
-build
+/build
apache_liminal.egg-info
scripts/*.tar.gz
scripts/*.whl
diff --git a/README.md b/README.md
index a92daff..c69f564 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,19 @@ volumes:
- volume: myvol1
local:
path: /Users/me/myvol1
+images:
+ - image: my_python_task_img
+ type: python
+ source: write_inputs
+ - image: my_parallelized_python_task_img
+ source: write_outputs
+ - image: my_server_image
+ type: python_server
+ source: myserver
+ endpoints:
+ - endpoint: /myendpoint1
+ module: my_server
+ function: myendpoint1func
pipelines:
- pipeline: my_pipeline
start_date: 1970-01-01
@@ -65,7 +78,6 @@ pipelines:
type: python
description: static input task
image: my_python_task_img
- source: write_inputs
env_vars:
NUM_FILES: 10
NUM_SPLITS: 3
@@ -78,7 +90,6 @@ pipelines:
type: python
description: parallelized python task
image: my_parallelized_python_task_img
- source: write_outputs
env_vars:
FOO: BAR
executors: 3
@@ -88,16 +99,9 @@ pipelines:
path: /mnt/vol1
cmd: python -u write_inputs.py
services:
- - service:
- name: my_python_server
- type: python_server
+ - service: my_python_server
description: my python server
image: my_server_image
- source: myserver
- endpoints:
- - endpoint: /myendpoint1
- module: my_server
- function: myendpoint1func
```
diff --git a/examples/aws-ml-app-demo/liminal.yml b/examples/aws-ml-app-demo/liminal.yml
index 03d1498..3284529 100644
--- a/examples/aws-ml-app-demo/liminal.yml
+++ b/examples/aws-ml-app-demo/liminal.yml
@@ -23,12 +23,9 @@ volumes:
claim_name: gettingstartedvol-pvc
local:
path: .
-services:
- - service:
- name: my_datascience_server
+images:
+ - image: myorg/mydatascienceapp
type: python_server
- description: my ds server
- image: myorg/mydatascienceapp
source: .
endpoints:
- endpoint: /predict
@@ -40,6 +37,9 @@ services:
- endpoint: /version
module: serving
function: version
+services:
+ - service: my_datascience_server
+ image: myorg/mydatascienceapp
pipelines:
- pipeline: my_datascience_pipeline
start_date: 1970-01-01
diff --git a/examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml b/examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml
index 1bfc21c..2361ee5 100644
--- a/examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml
+++ b/examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
apiVersion: v1
kind: Pod
metadata:
diff --git a/examples/liminal-getting-started/liminal.yml b/examples/liminal-getting-started/liminal.yml
index d8ad17b..bcbfd7d 100644
--- a/examples/liminal-getting-started/liminal.yml
+++ b/examples/liminal-getting-started/liminal.yml
@@ -22,6 +22,17 @@ volumes:
claim_name: gettingstartedvol-pvc
local:
path: .
+images:
+ - image: python_hello_world_example_image
+ type: python
+ source: helloworld
+ - image: liminal_getting_started_server_image
+ type: python_server
+ source: myserver
+ endpoints:
+ - endpoint: /myendpoint1
+ module: my_server
+ function: myendpoint1func
pipelines:
- pipeline: getting_started_pipeline
owner: Bosco Albert Baracus
@@ -41,7 +52,6 @@ pipelines:
type: python
description: static input task
image: python_hello_world_example_image
- source: helloworld
env_vars:
env1: "a"
env2: "b"
@@ -77,13 +87,6 @@ pipelines:
path: /mnt/gettingstartedvol
cmd: python -u hello_world.py
services:
- - service:
- name: liminal_getting_started_python_server
- type: python_server
+ - service: liminal_getting_started_python_server
description: my python server
image: liminal_getting_started_server_image
- source: myserver
- endpoints:
- - endpoint: /myendpoint1
- module: my_server
- function: myendpoint1func
diff --git a/liminal/build/service/python_server/Dockerfile b/liminal/build/image/python_server/Dockerfile
similarity index 100%
rename from liminal/build/service/python_server/Dockerfile
rename to liminal/build/image/python_server/Dockerfile
diff --git a/liminal/build/service/python_server/__init__.py b/liminal/build/image/python_server/__init__.py
similarity index 100%
rename from liminal/build/service/python_server/__init__.py
rename to liminal/build/image/python_server/__init__.py
diff --git a/liminal/build/service/python_server/liminal_python_server.py b/liminal/build/image/python_server/liminal_python_server.py
similarity index 100%
rename from liminal/build/service/python_server/liminal_python_server.py
rename to liminal/build/image/python_server/liminal_python_server.py
diff --git a/liminal/build/service/python_server/python_server.py b/liminal/build/image/python_server/python_server.py
similarity index 91%
rename from liminal/build/service/python_server/python_server.py
rename to liminal/build/image/python_server/python_server.py
index f0d5b99..25fcaa3 100644
--- a/liminal/build/service/python_server/python_server.py
+++ b/liminal/build/image/python_server/python_server.py
@@ -20,11 +20,10 @@ import os
import yaml
-from liminal.build.image_builder import ServiceImageBuilderMixin
from liminal.build.python import BasePythonImageBuilder
-class PythonServerImageBuilder(BasePythonImageBuilder, ServiceImageBuilderMixin):
+class PythonServerImageBuilder(BasePythonImageBuilder):
def __init__(self, config, base_path, relative_source_path, tag):
super().__init__(config, base_path, relative_source_path, tag)
diff --git a/liminal/build/service/python_server/python_server_requirements.txt b/liminal/build/image/python_server/python_server_requirements.txt
similarity index 100%
rename from liminal/build/service/python_server/python_server_requirements.txt
rename to liminal/build/image/python_server/python_server_requirements.txt
diff --git a/liminal/build/image_builder.py b/liminal/build/image_builder.py
index 11ff9f0..43b7267 100644
--- a/liminal/build/image_builder.py
+++ b/liminal/build/image_builder.py
@@ -150,7 +150,3 @@ class ImageBuilder:
overwrite with True to use docker buildkit
"""
return False
-
-
-class ServiceImageBuilderMixin(object):
- pass
diff --git a/liminal/build/liminal_apps_builder.py b/liminal/build/liminal_apps_builder.py
index d8297af..1b3fb8b 100644
--- a/liminal/build/liminal_apps_builder.py
+++ b/liminal/build/liminal_apps_builder.py
@@ -19,7 +19,7 @@
import logging
import os
-from liminal.build.image_builder import ImageBuilder, ServiceImageBuilderMixin
+from liminal.build.image_builder import ImageBuilder
from liminal.core.config.config import ConfigUtil
from liminal.core.util import files_util, class_util
@@ -33,50 +33,32 @@ def build_liminal_apps(path):
for liminal_config in configs:
base_path = os.path.dirname(files_util.resolve_pipeline_source_file(liminal_config['name']))
- if 'pipelines' in liminal_config:
- for pipeline in liminal_config['pipelines']:
- for task in pipeline['tasks']:
- task_name = task['task']
-
- if 'source' in task:
- task_type = task['type']
- builder_class = __get_task_build_class(task_type)
- if builder_class:
- __build_image(base_path, task, builder_class)
- else:
- raise ValueError(f'No such task type: {task_type}')
+ if 'images' in liminal_config:
+ for image in liminal_config['images']:
+ image_name = image['image']
+
+ if 'source' in image:
+ image_type = image['type']
+ builder_class = __get_image_builder_class(image_type)
+ if builder_class:
+ __build_image(base_path, image, builder_class)
else:
- logging.info(
- f'No source configured for task {task_name}, skipping build..')
-
- if 'services' in liminal_config:
- for service in liminal_config['services']:
- service_type = service['type']
- builder_class = __get_service_build_class(service_type)
- if builder_class:
- __build_image(base_path, service, builder_class)
+ raise ValueError(f'No such image type: {image_type}')
else:
- raise ValueError(f'No such service type: {service_type}')
+ logging.warning(f'No source configured for image {image_name}.')
def __build_image(base_path, builder_config, builder):
- if 'source' in builder_config:
- builder_instance = builder(
- config=builder_config,
- base_path=base_path,
- relative_source_path=builder_config['source'],
- tag=builder_config['image'])
- builder_instance.build()
- else:
- logging.info(f"No source provided for {builder_config['name']}, skipping.")
-
-
-def __get_task_build_class(task_type):
- return task_build_types.get(task_type, None)
+ builder_instance = builder(
+ config=builder_config,
+ base_path=base_path,
+ relative_source_path=builder_config['source'],
+ tag=builder_config['image'])
+ builder_instance.build()
-def __get_service_build_class(service_type):
- return service_build_types.get(service_type, None)
+def __get_image_builder_class(task_type):
+ return image_builder_types.get(task_type, None)
logging.info(f'Loading image builder implementations..')
@@ -85,19 +67,9 @@ logging.info(f'Loading image builder implementations..')
image_builders_package = 'liminal.build.image'
# user_image_builders_package = 'TODO: user_image_builders_package'
-task_build_types = class_util.find_subclasses_in_packages(
+image_builder_types = class_util.find_subclasses_in_packages(
[image_builders_package],
ImageBuilder)
-logging.info(f'Finished loading image builder implementations: {task_build_types}')
+logging.info(f'Finished loading image builder implementations: {image_builder_types}')
logging.info(f'Loading service image builder implementations..')
-
-# TODO: add configuration for user service image builders package
-service_builders_package = 'liminal.build.service'
-# user_service_builders_package = 'TODO: user_service_builders_package'
-
-service_build_types = class_util.find_subclasses_in_packages(
- [service_builders_package],
- ServiceImageBuilderMixin)
-
-logging.info(f'Finished loading service image builder implementations: {service_build_types}')
diff --git a/liminal/build/service/__init__.py b/liminal/build/service/__init__.py
deleted file mode 100644
index 217e5db..0000000
--- a/liminal/build/service/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/liminal/build/service/service_image_builder.py b/liminal/build/service/service_image_builder.py
deleted file mode 100644
index 3742bcc..0000000
--- a/liminal/build/service/service_image_builder.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
diff --git a/liminal/core/config/config.py b/liminal/core/config/config.py
index e43a870..51c7198 100644
--- a/liminal/core/config/config.py
+++ b/liminal/core/config/config.py
@@ -30,17 +30,19 @@ class ConfigUtil:
"""
Load and enrich config files under configs_path.
"""
- __BASE = "base"
- __PIPELINES = "pipelines"
- __SUPER = "super"
- __TYPE = "type"
- __SUB = "sub"
- __SERVICES = "services"
- __TASKS = "tasks"
- __PIPELINE_DEFAULTS = "pipeline_defaults"
- __BEFORE_TASKS = "before_tasks"
- __AFTER_TASKS = "after_tasks"
- __EXECUTORS = "executors"
+ __BASE = 'base'
+ __PIPELINES = 'pipelines'
+ __SUPER = 'super'
+ __TYPE = 'type'
+ __SUB = 'sub'
+ __SERVICES = 'services'
+ __TASKS = 'tasks'
+ __PIPELINE_DEFAULTS = 'pipeline_defaults'
+ __TASK_DEFAULTS = 'task_defaults'
+ __BEFORE_TASKS = 'before_tasks'
+ __AFTER_TASKS = 'after_tasks'
+ __EXECUTORS = 'executors'
+ __IMAGES = 'images'
def __init__(self, configs_path):
self.configs_path = configs_path
@@ -87,7 +89,8 @@ class ConfigUtil:
merged_superliminal = self.__merge_configs(supr, self.__get_superliminal(supr, soft_merge),
is_render_variables, soft_merge)
- sub[self.__EXECUTORS] = self.__merge_executors(sub, merged_superliminal)
+ sub[self.__EXECUTORS] = self.__merge_section(sub, merged_superliminal, self.__EXECUTORS)
+ sub[self.__IMAGES] = self.__merge_section(sub, merged_superliminal, self.__IMAGES)
if self.__is_subliminal(sub):
return self.__merge_sub_and_super(sub, merged_superliminal, is_render_variables)
@@ -160,9 +163,9 @@ class ConfigUtil:
files_util.dump_liminal_configs(liminal_configs=self.loaded_subliminals,
path=self.snapshot_path)
- def __merge_executors(self, subliminal, superliminal):
- return self.__deep_list_keyword_merge('executor', subliminal.get(self.__EXECUTORS, []),
- superliminal.get(self.__EXECUTORS, []))
+ def __merge_section(self, subliminal, superliminal, section):
+ return self.__deep_list_keyword_merge(section[:-1], subliminal.get(section, []),
+ superliminal.get(section, []))
@staticmethod
def __apply_pipeline_defaults(subliminal, superliminal, pipeline):
diff --git a/liminal/kubernetes/volume_util.py b/liminal/kubernetes/volume_util.py
index 9ca74d1..835bc0e 100644
--- a/liminal/kubernetes/volume_util.py
+++ b/liminal/kubernetes/volume_util.py
@@ -19,13 +19,15 @@
import logging
import os
import sys
+from time import sleep
from kubernetes import client, config
from kubernetes.client import V1PersistentVolume, V1PersistentVolumeClaim
+# noinspection PyBroadException
try:
config.load_kube_config()
-except:
+except Exception:
msg = "Kubernetes is not running\n"
sys.stdout.write(f"INFO: {msg}")
@@ -59,8 +61,12 @@ def create_local_volume(conf, namespace='default') -> None:
field_selector=f'metadata.name={name}'
).to_dict()['items']
- if not len(matching_volumes) > 0:
+ while len(matching_volumes) == 0:
_create_local_volume(conf, name)
+ sleep(5)
+ matching_volumes = _kubernetes.list_persistent_volume(
+ field_selector=f'metadata.name={name}'
+ ).to_dict()['items']
pvc_name = conf.get('claim_name', f'{name}-pvc')
@@ -68,35 +74,53 @@ def create_local_volume(conf, namespace='default') -> None:
field_selector=f'metadata.name={pvc_name}'
).to_dict()['items']
- if not len(matching_claims) > 0:
+ while len(matching_claims) == 0:
_create_persistent_volume_claim(pvc_name, name, namespace)
+ sleep(5)
+ matching_claims = _kubernetes.list_persistent_volume_claim_for_all_namespaces(
+ field_selector=f'metadata.name={pvc_name}'
+ ).to_dict()['items']
_LOCAL_VOLUMES.add(name)
def delete_local_volume(name, namespace='default'):
- matching_volumes = _kubernetes.list_persistent_volume(
- field_selector=f'metadata.name={name}'
- ).to_dict()['items']
-
- if len(matching_volumes) > 0:
- _LOG.info(f'Deleting persistent volume {name}')
- _kubernetes.delete_persistent_volume(name)
-
pvc_name = f'{name}-pvc'
- matching_claims = _kubernetes.list_persistent_volume_claim_for_all_namespaces(
- field_selector=f'metadata.name={pvc_name}'
- ).to_dict()['items']
+ matching_claims = _list_persistent_volume_claims(pvc_name)
if len(matching_claims) > 0:
_LOG.info(f'Deleting persistent volume claim {pvc_name}')
_kubernetes.delete_namespaced_persistent_volume_claim(pvc_name, namespace)
+ while len(matching_claims) > 0:
+ matching_claims = _list_persistent_volume_claims(pvc_name)
+
+ matching_volumes = _list_persistent_volumes(name)
+
+ if len(matching_volumes) > 0:
+ _LOG.info(f'Deleting persistent volume {name}')
+ _kubernetes.delete_persistent_volume(name)
+
+ while len(matching_volumes) > 0:
+ matching_volumes = _list_persistent_volumes(name)
+
if name in _LOCAL_VOLUMES:
_LOCAL_VOLUMES.remove(name)
+def _list_persistent_volume_claims(name):
+ return _kubernetes.list_persistent_volume_claim_for_all_namespaces(
+ field_selector=f'metadata.name={name}'
+ ).to_dict()['items']
+
+
+def _list_persistent_volumes(name):
+ return _kubernetes.list_persistent_volume(
+ field_selector=f'metadata.name={name}'
+ ).to_dict()['items']
+
+
def _create_persistent_volume_claim(pvc_name, volume_name, namespace):
_LOG.info(f'Creating persistent volume claim {pvc_name} with volume {volume_name}')
spec = {
diff --git a/tests/liminal/core/config/test_config.py b/tests/liminal/core/config/test_config.py
index e6da54a..b4baf9b 100644
--- a/tests/liminal/core/config/test_config.py
+++ b/tests/liminal/core/config/test_config.py
@@ -23,123 +23,133 @@ from liminal.core.config.config import ConfigUtil
# noinspection PyUnresolvedReferences,DuplicatedCode
-class TestHierarchicalConfig(TestCase):
+class TestConfigUtil(TestCase):
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.util.files_util.load')
def test_safe_load(self, find_config_files_mock):
subliminal = {
- "name": "my_subliminal_test",
- "type": "sub",
- "super": "my_superliminal_test",
- "pipelines": [
- {"name": "mypipe1", "param": "constant"},
- {"name": "mypipe2", "param": "constant"}
+ 'name': 'my_subliminal_test',
+ 'type': 'sub',
+ 'super': 'my_superliminal_test',
+ 'images': [{
+ 'image': 'my_image'
+ }],
+ 'pipelines': [
+ {'name': 'mypipe1', 'param': 'constant'},
+ {'name': 'mypipe2', 'param': 'constant'}
],
- "pipeline_defaults": {
- "param1": "param1_value"
+ 'pipeline_defaults': {
+ 'param1': 'param1_value'
},
- "task_defaults": {
- "job_start": {
- "task_sub_def": "task_sub_def_value"
+ 'task_defaults': {
+ 'job_start': {
+ 'task_sub_def': 'task_sub_def_value'
}
}
}
superliminal = {
- "name": "my_superliminal_test",
- "type": "super",
- "super": "super_superliminal",
- "pipeline_defaults": {
- "param2": "param2super_value",
- "param3": "param3super_value"
+ 'name': 'my_superliminal_test',
+ 'type': 'super',
+ 'super': 'super_superliminal',
+ 'images': [{
+ 'image': 'my_image',
+ 'source': '.'
+ }],
+ 'pipeline_defaults': {
+ 'param2': 'param2super_value',
+ 'param3': 'param3super_value'
},
- "task_defaults": {
- "job_start": {
- "task_def1": "task_def1_value",
- "task_def2": {
- "task_def2_1": "task_def2_1_value",
+ 'task_defaults': {
+ 'job_start': {
+ 'task_def1': 'task_def1_value',
+ 'task_def2': {
+ 'task_def2_1': 'task_def2_1_value',
}
}
}
}
super_superliminal = {
- "name": "super_superliminal",
- "type": "super",
- "pipeline_defaults": {
- "param2": "param2super_value",
- "param3": "param3hyper_value",
- "param4": "param4hyper_value"
+ 'name': 'super_superliminal',
+ 'type': 'super',
+ 'pipeline_defaults': {
+ 'param2': 'param2super_value',
+ 'param3': 'param3hyper_value',
+ 'param4': 'param4hyper_value'
}
}
- expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
- 'name': 'my_subliminal_test',
- 'pipeline_defaults': {'param1': 'param1_value'},
- 'pipelines': [{'description': 'add defaults parameters for all pipelines',
- 'name': 'mypipe1',
- 'param': 'constant',
- 'param1': 'param1_value',
- 'param2': 'param2super_value',
- 'param3': 'param3super_value',
- 'param4': 'param4hyper_value',
- 'tasks': [{'task': 'start',
- 'task_def1': 'task_def1_value',
- 'task_def2': {'task_def2_1': 'task_def2_1_value'},
- 'task_sub_def': 'task_sub_def_value',
- 'type': 'job_start'},
- {'task': 'end', 'type': 'job_end'}]},
- {'description': 'add defaults parameters for all pipelines',
- 'name': 'mypipe2',
- 'param': 'constant',
- 'param1': 'param1_value',
- 'param2': 'param2super_value',
- 'param3': 'param3super_value',
- 'param4': 'param4hyper_value',
- 'tasks': [{'task': 'start',
- 'task_def1': 'task_def1_value',
- 'task_def2': {'task_def2_1': 'task_def2_1_value'},
- 'task_sub_def': 'task_sub_def_value',
- 'type': 'job_start'},
- {'task': 'end', 'type': 'job_end'}]}],
- 'service_defaults': {'description': 'add defaults parameters for all '
- 'services'},
- 'services': [],
- 'super': 'my_superliminal_test',
- 'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
- 'type': 'sub'}]
+ expected = [{
+ 'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
+ 'name': 'my_subliminal_test',
+ 'pipeline_defaults': {'param1': 'param1_value'},
+ 'pipelines': [{'description': 'add defaults parameters for all pipelines',
+ 'name': 'mypipe1',
+ 'param': 'constant',
+ 'param1': 'param1_value',
+ 'param2': 'param2super_value',
+ 'param3': 'param3super_value',
+ 'param4': 'param4hyper_value',
+ 'tasks': [{'task': 'start',
+ 'task_def1': 'task_def1_value',
+ 'task_def2': {'task_def2_1': 'task_def2_1_value'},
+ 'task_sub_def': 'task_sub_def_value',
+ 'type': 'job_start'},
+ {'task': 'end', 'type': 'job_end'}]},
+ {'description': 'add defaults parameters for all pipelines',
+ 'name': 'mypipe2',
+ 'param': 'constant',
+ 'param1': 'param1_value',
+ 'param2': 'param2super_value',
+ 'param3': 'param3super_value',
+ 'param4': 'param4hyper_value',
+ 'tasks': [{'task': 'start',
+ 'task_def1': 'task_def1_value',
+ 'task_def2': {'task_def2_1': 'task_def2_1_value'},
+ 'task_sub_def': 'task_sub_def_value',
+ 'type': 'job_start'},
+ {'task': 'end', 'type': 'job_end'}]}],
+ 'service_defaults': {'description': 'add defaults parameters for all '
+ 'services'},
+ 'images': [{'image': 'my_image', 'source': '.'}],
+ 'services': [],
+ 'super': 'my_superliminal_test',
+ 'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
+ 'type': 'sub'
+ }]
find_config_files_mock.return_value = {
- "my_subliminal_test": subliminal,
- "my_superliminal_test": superliminal,
- "super_superliminal": super_superliminal
+ 'my_subliminal_test': subliminal,
+ 'my_superliminal_test': superliminal,
+ 'super_superliminal': super_superliminal
}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
# validate cache
self.assertEqual(expected, config_util.loaded_subliminals)
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.util.files_util.load')
def test_get_config(self, find_config_files_mock):
find_config_files_mock.return_value = {
- "my_subliminal_test": {
- "type": "sub"
+ 'my_subliminal_test': {
+ 'type': 'sub'
},
- "my_superliminal_test": {
- "type": "super"
+ 'my_superliminal_test': {
+ 'type': 'super'
}
}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
- self.assertEqual({"type": "sub"},
- config_util._ConfigUtil__get_config("my_subliminal_test"))
+ self.assertEqual({'type': 'sub'},
+ config_util._ConfigUtil__get_config('my_subliminal_test'))
- self.assertEqual({"type": "super"},
- config_util._ConfigUtil__get_config("my_superliminal_test"))
+ self.assertEqual({'type': 'super'},
+ config_util._ConfigUtil__get_config('my_superliminal_test'))
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.util.files_util.load')
def test_get_superliminal(self, find_config_files_mock):
base = {'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
'name': 'base',
@@ -154,15 +164,15 @@ class TestHierarchicalConfig(TestCase):
'python': {'executor': 'default_k8s'}},
'type': 'super'}
subliminal = {
- "name": "subliminal_test",
- "type": "sub"
+ 'name': 'subliminal_test',
+ 'type': 'sub'
}
find_config_files_mock.return_value = {
- "subliminal_test": subliminal
+ 'subliminal_test': subliminal
}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
self.assertEqual(base,
config_util._ConfigUtil__get_superliminal(subliminal, False))
@@ -171,219 +181,224 @@ class TestHierarchicalConfig(TestCase):
config_util._ConfigUtil__get_superliminal(base, False))
liminal = {
- "name": "subliminal_test",
- "type": "sub",
- "super": "my_superliminal"
+ 'name': 'subliminal_test',
+ 'type': 'sub',
+ 'super': 'my_superliminal'
}
with self.assertRaises(FileNotFoundError):
config_util._ConfigUtil__get_superliminal(liminal, False)
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.util.files_util.load')
def test_merge_superliminals(self, find_config_files_mock):
superliminal = {
- "name": "my_superliminal_test",
- "type": "super",
- "super": "super_superliminal",
- "pipeline_defaults": {
- "before_tasks": [
- {"task": "start-2", "type": "spark"},
+ 'name': 'my_superliminal_test',
+ 'type': 'super',
+ 'super': 'super_superliminal',
+ 'pipeline_defaults': {
+ 'before_tasks': [
+ {'task': 'start-2', 'type': 'spark'},
],
- "after_tasks": [
- {"task": "end-1", "type": "spark"}
+ 'after_tasks': [
+ {'task': 'end-1', 'type': 'spark'}
]
},
- "task_defaults": {
- "task_def1": "task_def1_value"
+ 'task_defaults': {
+ 'task_def1': 'task_def1_value'
}
}
super_superliminal = {
- "name": "super_superliminal",
- "type": "super",
- "pipeline_defaults": {
- "before_tasks": [
- {"task": "start-1", "type": "spark"}],
- "after_tasks": [
- {"task": "end-2", "type": "spark"}
+ 'name': 'super_superliminal',
+ 'type': 'super',
+ 'pipeline_defaults': {
+ 'before_tasks': [
+ {'task': 'start-1', 'type': 'spark'}],
+ 'after_tasks': [
+ {'task': 'end-2', 'type': 'spark'}
]
}
}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
find_config_files_mock.return_value = {
- "super_superliminal": super_superliminal,
- "superliminal": superliminal
+ 'super_superliminal': super_superliminal,
+ 'superliminal': superliminal
}
- expected = {'name': 'my_superliminal_test',
- 'pipeline_defaults': {'after_tasks': [{'task': 'end-1', 'type': 'spark'},
- {'task': 'end-2', 'type': 'spark'}],
- 'before_tasks': [{'task': 'start-1', 'type': 'spark'},
- {'task': 'start-2', 'type': 'spark'}]},
- 'super': 'super_superliminal',
- 'task_defaults': {'task_def1': 'task_def1_value'},
- 'type': 'super'}
+ expected = {
+ 'name': 'my_superliminal_test',
+ 'pipeline_defaults': {'after_tasks': [{'task': 'end-1', 'type': 'spark'},
+ {'task': 'end-2', 'type': 'spark'}],
+ 'before_tasks': [{'task': 'start-1', 'type': 'spark'},
+ {'task': 'start-2', 'type': 'spark'}]},
+ 'super': 'super_superliminal',
+ 'task_defaults': {'task_def1': 'task_def1_value'},
+ 'type': 'super'
+ }
self.assertEqual(expected,
dict(config_util._ConfigUtil__merge_superliminals(superliminal,
super_superliminal)))
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.util.files_util.load')
@mock.patch.dict(os.environ, {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True'})
def test_safe_load_with_variables(self, find_config_files_mock):
subliminal = {
- "name": "my_subliminal_test",
- "type": "sub",
- "super": "my_superliminal_test",
- "variables": {
- "var": "simple case",
- "var-2": "-case",
- "var_2": "_case",
- "image": "prod image",
- "a": "{{env}}1",
- "b": "{{a}}2",
- "c": "{{a}}{{b}}2"
+ 'name': 'my_subliminal_test',
+ 'type': 'sub',
+ 'super': 'my_superliminal_test',
+ 'variables': {
+ 'var': 'simple case',
+ 'var-2': '-case',
+ 'var_2': '_case',
+ 'image': 'prod image',
+ 'a': '{{env}}1',
+ 'b': '{{a}}2',
+ 'c': '{{a}}{{b}}2'
},
- "pipelines": [
- {"name": "mypipe1", "param": "{{var}}",
- "tasks": [
+ 'pipelines': [
+ {'name': 'mypipe1', 'param': '{{var}}',
+ 'tasks': [
{'task': 'sub_tasks',
'type': 'dummy'},
]},
- {"name": "mypipe2", "param": "{{var-2 }}", "tasks": [
+ {'name': 'mypipe2', 'param': '{{var-2 }}', 'tasks': [
{'task': 'sub_tasks',
'type': 'dummy'},
]}
],
- "pipeline_defaults": {
- "param1": "{{var-2}}"
+ 'pipeline_defaults': {
+ 'param1': '{{var-2}}'
},
- "task_defaults": {
- "job_start": {
- "task_def1:": "task_sub_def_value"
+ 'task_defaults': {
+ 'job_start': {
+ 'task_def1:': 'task_sub_def_value'
}
},
- "services": [
+ 'services': [
{
- "name": "my_python_server",
- "type": "python_server",
- "image": "{{image}}"
+ 'name': 'my_python_server',
+ 'type': 'python_server',
+ 'image': '{{image}}'
},
{
- "name": "my_python_server_for_stg",
- "type": "python_server",
- "image": "{{default_image}}"
+ 'name': 'my_python_server_for_stg',
+ 'type': 'python_server',
+ 'image': '{{default_image}}'
}
]}
superliminal = {
- "name": "my_superliminal_test",
- "type": "super",
- "variables": {
- "var-2": "override",
- "var3": "super_var",
- "default_image": "default_image_value",
- "image": "default_image_value"
+ 'name': 'my_superliminal_test',
+ 'type': 'super',
+ 'variables': {
+ 'var-2': 'override',
+ 'var3': 'super_var',
+ 'default_image': 'default_image_value',
+ 'image': 'default_image_value'
},
- "super": "super_superliminal",
- "pipeline_defaults": {
- "param2": "{{pipe-var}}",
- "param3": "param3super_value",
- "before_tasks": [
+ 'super': 'super_superliminal',
+ 'pipeline_defaults': {
+ 'param2': '{{pipe-var}}',
+ 'param3': 'param3super_value',
+ 'before_tasks': [
{'task': 'second_task', 'type': 'dummy'},
]
},
- "task_defaults": {
- "pipeline": {
- "path": "{{var-2}}",
- "task_def1": "task_def1_value",
- "task_def2": {
- "task_def2_1": "task_def2_1_value",
+ 'task_defaults': {
+ 'pipeline': {
+ 'path': '{{var-2}}',
+ 'task_def1': 'task_def1_value',
+ 'task_def2': {
+ 'task_def2_1': 'task_def2_1_value',
}
}
}
}
super_superliminal = {
- "name": "super_superliminal",
- "type": "super",
- "variables": {
- "default_image": "def_default_image_value"
+ 'name': 'super_superliminal',
+ 'type': 'super',
+ 'variables': {
+ 'default_image': 'def_default_image_value'
},
- "pipeline_defaults": {
- "global_conf": "{{var3}}",
- "param2": "param2super_value",
- "param3": "param3hyper_value",
- "param4": "param4hyper_value",
- "after_tasks": [
+ 'pipeline_defaults': {
+ 'global_conf': '{{var3}}',
+ 'param2': 'param2super_value',
+ 'param3': 'param3hyper_value',
+ 'param4': 'param4hyper_value',
+ 'after_tasks': [
{'task': 'before_last_task', 'type': 'dummy'},
]
}
}
- expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
- 'name': 'my_subliminal_test',
- 'pipeline_defaults': {'param1': '-case'},
- 'pipelines': [{'description': 'add defaults parameters for all pipelines',
- 'global_conf': 'super_var',
- 'name': 'mypipe1',
- 'param': 'simple case',
- 'param1': '-case',
- 'param2': '{{pipe-var}}',
- 'param3': 'param3super_value',
- 'param4': 'param4hyper_value',
- 'tasks': [{'task': 'start',
- 'task_def1:': 'task_sub_def_value',
- 'type': 'job_start'},
- {'task': 'second_task', 'type': 'dummy'},
- {'task': 'sub_tasks', 'type': 'dummy'},
- {'task': 'before_last_task', 'type': 'dummy'},
- {'task': 'end', 'type': 'job_end'}]},
- {'description': 'add defaults parameters for all pipelines',
- 'global_conf': 'super_var',
- 'name': 'mypipe2',
- 'param': '-case',
- 'param1': '-case',
- 'param2': '{{pipe-var}}',
- 'param3': 'param3super_value',
- 'param4': 'param4hyper_value',
- 'tasks': [{'task': 'start',
- 'task_def1:': 'task_sub_def_value',
- 'type': 'job_start'},
- {'task': 'second_task', 'type': 'dummy'},
- {'task': 'sub_tasks', 'type': 'dummy'},
- {'task': 'before_last_task', 'type': 'dummy'},
- {'task': 'end', 'type': 'job_end'}]}],
- 'service_defaults': {'description': 'add defaults parameters for all '
- 'services'},
- 'services': [{'description': 'add defaults parameters for all services',
- 'image': 'prod image',
- 'name': 'my_python_server',
- 'type': 'python_server'},
- {'description': 'add defaults parameters for all services',
- 'image': 'default_image_value',
- 'name': 'my_python_server_for_stg',
- 'type': 'python_server'}],
- 'super': 'my_superliminal_test',
- 'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
- 'type': 'sub',
- 'variables': {'a': 'myenv1',
- 'b': 'myenv12',
- 'c': 'myenv1myenv122',
- 'image': 'prod image',
- 'var': 'simple case',
- 'var-2': '-case',
- 'var_2': '_case'}}]
+ expected = [{
+ 'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
+ 'name': 'my_subliminal_test',
+ 'pipeline_defaults': {'param1': '-case'},
+ 'pipelines': [{'description': 'add defaults parameters for all pipelines',
+ 'global_conf': 'super_var',
+ 'name': 'mypipe1',
+ 'param': 'simple case',
+ 'param1': '-case',
+ 'param2': '{{pipe-var}}',
+ 'param3': 'param3super_value',
+ 'param4': 'param4hyper_value',
+ 'tasks': [{'task': 'start',
+ 'task_def1:': 'task_sub_def_value',
+ 'type': 'job_start'},
+ {'task': 'second_task', 'type': 'dummy'},
+ {'task': 'sub_tasks', 'type': 'dummy'},
+ {'task': 'before_last_task', 'type': 'dummy'},
+ {'task': 'end', 'type': 'job_end'}]},
+ {'description': 'add defaults parameters for all pipelines',
+ 'global_conf': 'super_var',
+ 'name': 'mypipe2',
+ 'param': '-case',
+ 'param1': '-case',
+ 'param2': '{{pipe-var}}',
+ 'param3': 'param3super_value',
+ 'param4': 'param4hyper_value',
+ 'tasks': [{'task': 'start',
+ 'task_def1:': 'task_sub_def_value',
+ 'type': 'job_start'},
+ {'task': 'second_task', 'type': 'dummy'},
+ {'task': 'sub_tasks', 'type': 'dummy'},
+ {'task': 'before_last_task', 'type': 'dummy'},
+ {'task': 'end', 'type': 'job_end'}]}],
+ 'service_defaults': {'description': 'add defaults parameters for all '
+ 'services'},
+ 'images': [],
+ 'services': [{'description': 'add defaults parameters for all services',
+ 'image': 'prod image',
+ 'name': 'my_python_server',
+ 'type': 'python_server'},
+ {'description': 'add defaults parameters for all services',
+ 'image': 'default_image_value',
+ 'name': 'my_python_server_for_stg',
+ 'type': 'python_server'}],
+ 'super': 'my_superliminal_test',
+ 'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
+ 'type': 'sub',
+ 'variables': {'a': 'myenv1',
+ 'b': 'myenv12',
+ 'c': 'myenv1myenv122',
+ 'image': 'prod image',
+ 'var': 'simple case',
+ 'var-2': '-case',
+ 'var_2': '_case'}
+ }]
find_config_files_mock.return_value = {
- "my_subliminal_test": subliminal,
- "my_superliminal_test": superliminal,
- "super_superliminal": super_superliminal
+ 'my_subliminal_test': subliminal,
+ 'my_superliminal_test': superliminal,
+ 'super_superliminal': super_superliminal
}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
self.assertEqual(expected, config_util.safe_load(is_render_variables=True))
@@ -391,34 +406,35 @@ class TestHierarchicalConfig(TestCase):
self.assertEqual(expected, config_util.loaded_subliminals)
@mock.patch('os.path.exists')
- @mock.patch("liminal.core.environment.get_airflow_home_dir")
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.environment.get_airflow_home_dir')
+ @mock.patch('liminal.core.util.files_util.load')
@mock.patch.dict(os.environ, {'LIMINAL_STAND_ALONE_MODE': 'True', 'POD_NAMESPACE': 'my_pod_ns'})
def test_liminal_config_snapshot(self, find_config_files_mock,
get_airflow_dir_mock, path_exists_mock):
subliminal = {
- "name": "my_subliminal_test",
- "type": "sub",
- "variables": {
- "var": 1,
- "var-2": True
+ 'name': 'my_subliminal_test',
+ 'type': 'sub',
+ 'variables': {
+ 'var': 1,
+ 'var-2': True
},
- "pipelines": [
- {"name": "mypipe1", "param": "{{var}}"},
- {"name": "mypipe2", "param": "{{var-2 }}"}
+ 'pipelines': [
+ {'name': 'mypipe1', 'param': '{{var}}'},
+ {'name': 'mypipe2', 'param': '{{var-2 }}'}
]
}
- expected = {'name': 'my_subliminal_test', 'type': 'sub',
- 'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
- 'service_defaults': {'description': 'add defaults parameters for all services'},
- 'task_defaults': {
- 'description': 'add defaults parameters for all tasks separate by task type',
- 'python': {'executor': 'default_k8s'}}, 'pipeline_defaults': {
+ expected = {
+ 'name': 'my_subliminal_test', 'type': 'sub',
+ 'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
+ 'service_defaults': {'description': 'add defaults parameters for all services'},
+ 'task_defaults': {
+ 'description': 'add defaults parameters for all tasks separate by task type',
+ 'python': {'executor': 'default_k8s'}}, 'pipeline_defaults': {
'description': 'add defaults parameters for all pipelines',
'before_tasks': [{'task': 'start', 'type': 'job_start'}],
'after_tasks': [{'task': 'end', 'type': 'job_end'}]},
- 'variables': {'var': 1, 'var-2': True}, 'pipelines': [
+ 'variables': {'var': 1, 'var-2': True}, 'pipelines': [
{'name': 'mypipe1', 'param': '1',
'description': 'add defaults parameters for all pipelines',
'tasks': [{'task': 'start', 'type': 'job_start'},
@@ -426,18 +442,21 @@ class TestHierarchicalConfig(TestCase):
{'name': 'mypipe2', 'param': 'True',
'description': 'add defaults parameters for all pipelines',
'tasks': [{'task': 'start', 'type': 'job_start'},
- {'task': 'end', 'type': 'job_end'}]}], 'services': []}
+ {'task': 'end', 'type': 'job_end'}]}],
+ 'images': [],
+ 'services': []
+ }
find_config_files_mock.return_value = {
- "my_subliminal_test": subliminal
+ 'my_subliminal_test': subliminal
}
- get_airflow_dir_mock.return_value = "/tmp"
+ get_airflow_dir_mock.return_value = '/tmp'
path_exists_mock.return_value = True
- with mock.patch("builtins.open", mock.mock_open()) as m:
- with mock.patch("yaml.dump") as ydm:
- config_util = ConfigUtil("")
+ with mock.patch('builtins.open', mock.mock_open()) as m:
+ with mock.patch('yaml.dump') as ydm:
+ config_util = ConfigUtil('')
config_util.safe_load(is_render_variables=True)
config_util.snapshot_final_liminal_configs()
@@ -445,28 +464,28 @@ class TestHierarchicalConfig(TestCase):
os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w')
ydm.assert_called_once_with(expected, m.return_value, default_flow_style=False)
- @mock.patch("liminal.core.util.files_util.load")
+ @mock.patch('liminal.core.util.files_util.load')
def test_soft_merge_load(self, find_config_files_mock):
subliminal = {
- "name": "my_name",
- "type": "sub",
- "super": "my_super"
+ 'name': 'my_name',
+ 'type': 'sub',
+ 'super': 'my_super'
}
- find_config_files_mock.return_value = {"my_subliminal_test": subliminal}
+ find_config_files_mock.return_value = {'my_subliminal_test': subliminal}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
self.assertEqual([subliminal],
config_util.safe_load(is_render_variables=True, soft_merge=True))
def test_non_soft_merge_load(self):
subliminal = {
- "name": "my_name",
- "type": "sub",
- "super": "my_super"
+ 'name': 'my_name',
+ 'type': 'sub',
+ 'super': 'my_super'
}
- config_util = ConfigUtil("")
+ config_util = ConfigUtil('')
self.assertRaises(FileNotFoundError,
config_util._ConfigUtil__get_superliminal,
diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
index 88f9663..f8173e9 100644
--- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
+++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
@@ -28,7 +28,9 @@ from unittest import TestCase
import docker
from liminal.build.python import PythonImageVersions
-from liminal.build.service.python_server.python_server import PythonServerImageBuilder
+from liminal.build.image.python_server.python_server import PythonServerImageBuilder
+
+IMAGE_NAME = 'liminal_server_image'
class TestPythonServer(TestCase):
@@ -36,8 +38,7 @@ class TestPythonServer(TestCase):
def setUp(self) -> None:
super().setUp()
self.docker_client = docker.from_env()
- self.config = self.__create_conf('my_task')
- self.image_name = self.config['image']
+ self.config = self.__create_conf()
self.__remove_containers()
def tearDown(self) -> None:
@@ -45,7 +46,7 @@ class TestPythonServer(TestCase):
self.docker_client.close()
def test_build_python_server(self):
- versions = [None] + list(PythonImageVersions().supported_versions)
+ versions = list(PythonImageVersions().supported_versions)
for version in versions:
build_out = self.__test_build_python_server(python_version=version)
self.assertTrue('RUN pip install -r requirements.txt' in build_out,
@@ -58,11 +59,10 @@ class TestPythonServer(TestCase):
'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf pip install' in build_out,
'Incorrect pip command')
- def __test_build_python_server(self, use_pip_conf=False,
- python_version=None):
+ def __test_build_python_server(self, use_pip_conf=False, python_version=None):
base_path = os.path.join(os.path.dirname(__file__), '../../../liminal')
- config = self.__create_conf('my_task')
+ config = self.__create_conf()
if use_pip_conf:
config['pip_conf'] = os.path.join(base_path, 'pip.conf')
@@ -73,12 +73,11 @@ class TestPythonServer(TestCase):
builder = PythonServerImageBuilder(config=config,
base_path=base_path,
relative_source_path='myserver',
- tag=self.image_name)
+ tag=IMAGE_NAME)
build_out = str(builder.build())
- thread = threading.Thread(target=self.__run_container, args=[self.image_name])
- thread.daemon = True
+ thread = threading.Thread(target=self.__run_container)
thread.start()
time.sleep(5)
@@ -98,13 +97,14 @@ class TestPythonServer(TestCase):
self.assertEqual(f'Input was: {json.loads(json_string)}', server_response)
+ self.__remove_containers()
+
return build_out
def __remove_containers(self):
- logging.info(f'Stopping containers with image: {self.image_name}')
+ logging.info(f'Stopping containers with image: {IMAGE_NAME}')
- all_containers = self.docker_client.containers
- matching_containers = all_containers.list(filters={'ancestor': self.image_name})
+ matching_containers = self.__get_docker_containers()
for container in matching_containers:
container_id = container.id
@@ -113,22 +113,28 @@ class TestPythonServer(TestCase):
logging.info(f'Removing container {container_id}')
self.docker_client.api.remove_container(container_id)
- self.docker_client.containers.prune()
+ while len(matching_containers) > 0:
+ matching_containers = self.__get_docker_containers()
+
+ def __get_docker_containers(self):
+ return self.docker_client.containers.list(
+ filters={'ancestor': IMAGE_NAME}
+ )
- def __run_container(self, image_name):
+ def __run_container(self):
try:
- logging.info(f'Running container for image: {image_name}')
- self.docker_client.containers.run(image_name, ports={'80/tcp': 9294})
+ logging.info(f'Running container for image: {IMAGE_NAME}')
+ self.docker_client.containers.run(IMAGE_NAME, ports={'80/tcp': 9294},
+ detach=True)
except Exception as err:
logging.exception(err)
pass
@staticmethod
- def __create_conf(task_id):
+ def __create_conf():
return {
- 'task': task_id,
+ 'image': IMAGE_NAME,
'cmd': 'foo bar',
- 'image': 'liminal_server_image',
'source': 'baz',
'input_type': 'my_input_type',
'input_path': 'my_input',
diff --git a/tests/runners/airflow/liminal/liminal.yml b/tests/runners/airflow/liminal/liminal.yml
index ce8a9d8..48a9907 100644
--- a/tests/runners/airflow/liminal/liminal.yml
+++ b/tests/runners/airflow/liminal/liminal.yml
@@ -21,6 +21,23 @@ volumes:
- volume: myvol1
local:
path: /tmp/liminal_tests
+images:
+ - image: my_python_task_img
+ type: python
+ source: write_inputs
+ no_cache: True
+ - image: my_parallelized_python_task_img
+ type: python
+ source: write_outputs
+ no_cache: True
+ - image: my_server_image
+ type: python_server
+ source: myserver
+ no_cache: True
+ endpoints:
+ - endpoint: /myendpoint1
+ module: my_server
+ function: myendpoint1func
pipelines:
- pipeline: my_pipeline
owner: Bosco Albert Baracus
@@ -40,7 +57,6 @@ pipelines:
type: python
description: static input task
image: my_python_task_img
- source: write_inputs
env_vars:
NUM_FILES: 10
NUM_SPLITS: 3
@@ -53,7 +69,6 @@ pipelines:
type: python
description: parallelized python task
image: my_parallelized_python_task_img
- source: write_outputs
env_vars:
FOO: BAR
executors: 3
@@ -63,13 +78,6 @@ pipelines:
path: /mnt/vol1
cmd: python -u write_inputs.py
services:
- - service:
- name: my_python_server
- type: python_server
+ - service: my_python_server
description: my python server
image: my_server_image
- source: myserver
- endpoints:
- - endpoint: /myendpoint1
- module: my_server
- function: myendpoint1func
diff --git a/tests/runners/apps/test_app/extra/liminal.yml b/tests/runners/apps/test_app/extra/liminal.yml
index daef21e..546d16a 100644
--- a/tests/runners/apps/test_app/extra/liminal.yml
+++ b/tests/runners/apps/test_app/extra/liminal.yml
@@ -28,9 +28,11 @@ images:
- image: my_static_input_task_image
type: python
source: helloworld
+ no_cache: True
- image: my_task_output_input_task_image
type: python
source: helloworld
+ no_cache: True
pipeline_defaults:
tasks:
- task: my_static_input_task
diff --git a/tests/runners/apps/test_app/liminal.yml b/tests/runners/apps/test_app/liminal.yml
index 9c8e52c..5a63b0f 100644
--- a/tests/runners/apps/test_app/liminal.yml
+++ b/tests/runners/apps/test_app/liminal.yml
@@ -22,9 +22,11 @@ images:
- image: my_static_input_task_image
type: python
source: helloworld
+ no_cache: True
- image: my_server_image
- type: python
+ type: python_server
source: myserver
+ no_cache: True
pipelines:
- pipeline: my_pipeline
owner: Bosco Albert Baracus
@@ -44,8 +46,6 @@ pipelines:
executors: 2
cmd: python -u hello_world.py
services:
- - service:
- name: my_python_server
- type: python_server
+ - service: my_python_server
description: my python server
image: my_server_image