You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@liminal.apache.org by jb...@apache.org on 2020/07/20 06:24:52 UTC
[incubator-liminal] 12/43: Elaborate build tests
This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit eed1cd0b445f0878a9b078b7c206c54e47244bd6
Author: aviemzur <av...@gmail.com>
AuthorDate: Thu Mar 12 11:30:55 2020 +0200
Elaborate build tests
---
rainbow/build/build_rainbow.py | 4 +--
rainbow/docker/python/python_image.py | 2 --
.../airflow/build/python/test_python_image.py | 37 +++++++++++++++-------
tests/runners/airflow/build/test_build_rainbow.py | 21 +++++++++++-
tests/runners/airflow/rainbow/rainbow.yml | 4 +--
tests/runners/airflow/tasks/test_python.py | 2 +-
6 files changed, 51 insertions(+), 19 deletions(-)
diff --git a/rainbow/build/build_rainbow.py b/rainbow/build/build_rainbow.py
index 280b862..7c03104 100644
--- a/rainbow/build/build_rainbow.py
+++ b/rainbow/build/build_rainbow.py
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
import os
import yaml
@@ -46,10 +47,9 @@ def build_rainbow(path):
tag=task['image'])
-# TODO: task class registry
+# TODO: build class registry
build_classes = {
'python': PythonImage
-
}
diff --git a/rainbow/docker/python/python_image.py b/rainbow/docker/python/python_image.py
index 2cd3594..ae7bc23 100644
--- a/rainbow/docker/python/python_image.py
+++ b/rainbow/docker/python/python_image.py
@@ -56,8 +56,6 @@ class PythonImage:
for file in extra_files + [dockerfile_path]:
self.__copy_file(file, temp_dir)
- print(temp_dir, os.listdir(temp_dir))
-
docker_client = docker.from_env()
docker_client.images.build(path=temp_dir, tag=tag)
diff --git a/tests/runners/airflow/build/python/test_python_image.py b/tests/runners/airflow/build/python/test_python_image.py
index c290720..a8c02b6 100644
--- a/tests/runners/airflow/build/python/test_python_image.py
+++ b/tests/runners/airflow/build/python/test_python_image.py
@@ -15,24 +15,39 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from unittest import TestCase
import docker
-from rainbow.docker.python import python_image
+from rainbow.docker.python.python_image import PythonImage
-def test_build(self):
- config = self.__create_conf('my_task')
+class TestPythonImage(TestCase):
- image_name = config['image']
+ def test_build(self):
+ config = self.__create_conf('my_task')
- python_image.build('tests/runners/airflow/rainbow', 'hello_world', 'image_name')
+ image_name = config['image']
- # TODO: elaborate test of image, validate input/output
+ PythonImage().build('tests/runners/airflow/rainbow', 'hello_world', 'image_name')
- docker_client = docker.from_env()
- docker_client.images.get(image_name)
- container_log = docker_client.containers.run(image_name, "python hello_world.py")
- docker_client.close()
+ # TODO: elaborate test of image, validate input/output
- self.assertEqual("b'Hello world!\\n'", str(container_log))
+ docker_client = docker.from_env()
+ docker_client.images.get(image_name)
+ container_log = docker_client.containers.run(image_name, "python hello_world.py")
+ docker_client.close()
+
+ self.assertEqual("b'Hello world!\\n'", str(container_log))
+
+ @staticmethod
+ def __create_conf(task_id):
+ return {
+ 'task': task_id,
+ 'cmd': 'foo bar',
+ 'image': 'rainbow_image',
+ 'source': 'tests/runners/airflow/rainbow/hello_world',
+ 'input_type': 'my_input_type',
+ 'input_path': 'my_input',
+ 'output_path': '/my_output.json'
+ }
diff --git a/tests/runners/airflow/build/test_build_rainbow.py b/tests/runners/airflow/build/test_build_rainbow.py
index c8fec6e..d1b28aa 100644
--- a/tests/runners/airflow/build/test_build_rainbow.py
+++ b/tests/runners/airflow/build/test_build_rainbow.py
@@ -1,8 +1,27 @@
+import unittest
from unittest import TestCase
+import docker
from rainbow.build import build_rainbow
-class Test(TestCase):
+class TestBuildRainbow(TestCase):
+
def test_build_rainbow(self):
+ docker_client = docker.client.from_env()
+ image_names = ['rainbow_image', 'rainbow_image2']
+
+ for image_name in image_names:
+ if len(docker_client.images.list(image_name)) > 0:
+ docker_client.images.remove(image=image_name)
+
build_rainbow.build_rainbow('tests/runners/airflow/rainbow')
+
+ for image_name in image_names:
+ docker_client.images.get(name=image_name)
+
+ docker_client.close()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/runners/airflow/rainbow/rainbow.yml b/tests/runners/airflow/rainbow/rainbow.yml
index fd30028..1a834d7 100644
--- a/tests/runners/airflow/rainbow/rainbow.yml
+++ b/tests/runners/airflow/rainbow/rainbow.yml
@@ -28,7 +28,7 @@ pipelines:
- task: my_static_config_task
type: python
description: my 1st ds task
- image: my_image
+ image: rainbow_image
source: hello_world
env_vars:
env1: "a"
@@ -40,7 +40,7 @@ pipelines:
- task: my_static_config_task2
type: python
description: my 1st ds task
- image: mytask1artifactid
+ image: rainbow_image2
source: hello_world
env_vars:
env1: "a"
diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py
index 37a325a..8477c69 100644
--- a/tests/runners/airflow/tasks/test_python.py
+++ b/tests/runners/airflow/tasks/test_python.py
@@ -51,7 +51,7 @@ class TestPythonTask(TestCase):
return {
'task': task_id,
'cmd': 'foo bar',
- 'image': 'my_image',
+ 'image': 'rainbow_image',
'source': 'tests/runners/airflow/rainbow/hello_world',
'input_type': 'my_input_type',
'input_path': 'my_input',