You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@liminal.apache.org by jb...@apache.org on 2020/07/20 06:25:12 UTC
[incubator-liminal] 32/43: Use user pip conf in docker build
This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 8df40b2cb91eb2428e58c58e7dad9a290f9c185e
Author: aviem-naturalint <av...@naturalint.com>
AuthorDate: Sat Apr 11 08:14:08 2020 +0300
Use user pip conf in docker build
---
rainbow/build/build_rainbows.py | 4 +-
rainbow/build/image/python/Dockerfile | 26 ++++++-
rainbow/build/image/python/container-setup.sh | 2 +
rainbow/build/image/python/container-teardown.sh | 4 +-
rainbow/build/image/python/python.py | 7 +-
rainbow/build/{image => }/image_builder.py | 70 ++++++++++++-----
rainbow/build/python.py | 74 ++++++++++++++++++
rainbow/build/service/python_server/Dockerfile | 26 ++++++-
.../build/service/python_server/python_server.py | 8 +-
.../kubernetes_pod_operator_with_input_output.py | 5 +-
run_tests.sh | 4 +-
.../python/test_python_server_image_builder.py | 36 +++++++--
.../build/python/test_python_image_builder.py | 90 ++++++++++++++++++----
tests/runners/airflow/build/test_build_rainbows.py | 2 +-
.../airflow/rainbow/helloworld/hello_world.py | 10 ++-
.../{helloworld/hello_world.py => pip.conf} | 10 ---
tests/runners/airflow/rainbow/rainbow.yml | 8 +-
17 files changed, 308 insertions(+), 78 deletions(-)
diff --git a/rainbow/build/build_rainbows.py b/rainbow/build/build_rainbows.py
index 4ed5bab..b7ea6eb 100644
--- a/rainbow/build/build_rainbows.py
+++ b/rainbow/build/build_rainbows.py
@@ -20,13 +20,13 @@ import os
import yaml
-from rainbow.build.image.image_builder import ImageBuilder, ServiceImageBuilderMixin
+from rainbow.build.image_builder import ImageBuilder, ServiceImageBuilderMixin
from rainbow.core.util import files_util, class_util
def build_rainbows(path):
"""
- TODO: doc for build_rainbows
+ Build images for rainbows in path.
"""
config_files = files_util.find_config_files(path)
diff --git a/rainbow/build/image/python/Dockerfile b/rainbow/build/image/python/Dockerfile
index d4e3ed2..8e4de05 100644
--- a/rainbow/build/image/python/Dockerfile
+++ b/rainbow/build/image/python/Dockerfile
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
# Use an official Python runtime as a parent image
FROM python:3.7-slim
@@ -11,9 +29,11 @@ WORKDIR /app
# Be careful when changing this code. !
# Install any needed packages specified in requirements.txt
-COPY ./requirements.txt /app
-RUN pip install -r requirements.txt
+COPY ./requirements.txt /app/
+
+# mount the secret in the correct location, then run pip install
+RUN {{mount}} pip install -r requirements.txt
# Copy the current directory contents into the container at /app
RUN echo "Copying source code.."
-COPY . /app
+COPY . /app/
diff --git a/rainbow/build/image/python/container-setup.sh b/rainbow/build/image/python/container-setup.sh
index 883f1e1..c9e5cef 100755
--- a/rainbow/build/image/python/container-setup.sh
+++ b/rainbow/build/image/python/container-setup.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+echo 'Writing rainbow input..'
+
echo """$RAINBOW_INPUT""" > /rainbow_input.json
AIRFLOW_RETURN_FILE=/airflow/xcom/return.json
diff --git a/rainbow/build/image/python/container-teardown.sh b/rainbow/build/image/python/container-teardown.sh
index ef213a8..46c4426 100755
--- a/rainbow/build/image/python/container-teardown.sh
+++ b/rainbow/build/image/python/container-teardown.sh
@@ -1,6 +1,8 @@
#!/bin/sh
+echo 'Writing rainbow output..'
+
USER_CONFIG_OUTPUT_FILE=$1
if [ "$USER_CONFIG_OUTPUT_FILE" != "" ]; then
- cp ${USER_CONFIG_OUTPUT_FILE} /airflow/xcom/return.json
+ cp "${USER_CONFIG_OUTPUT_FILE}" /airflow/xcom/return.json
fi
diff --git a/rainbow/build/image/python/python.py b/rainbow/build/image/python/python.py
index f4fb03b..0ecec77 100644
--- a/rainbow/build/image/python/python.py
+++ b/rainbow/build/image/python/python.py
@@ -18,10 +18,10 @@
import os
-from rainbow.build.image.image_builder import ImageBuilder
+from rainbow.build.python import BasePythonImageBuilder
-class PythonImageBuilder(ImageBuilder):
+class PythonImageBuilder(BasePythonImageBuilder):
def __init__(self, config, base_path, relative_source_path, tag):
super().__init__(config, base_path, relative_source_path, tag)
@@ -30,8 +30,7 @@ class PythonImageBuilder(ImageBuilder):
def _dockerfile_path():
return os.path.join(os.path.dirname(__file__), 'Dockerfile')
- @staticmethod
- def _additional_files_from_paths():
+ def _additional_files_from_paths(self):
return [
os.path.join(os.path.dirname(__file__), 'container-setup.sh'),
os.path.join(os.path.dirname(__file__), 'container-teardown.sh'),
diff --git a/rainbow/build/image/image_builder.py b/rainbow/build/image_builder.py
similarity index 64%
rename from rainbow/build/image/image_builder.py
rename to rainbow/build/image_builder.py
index e716b9d..a56a22e 100644
--- a/rainbow/build/image/image_builder.py
+++ b/rainbow/build/image_builder.py
@@ -18,24 +18,23 @@
import os
import shutil
+import subprocess
import tempfile
-import docker
-
class ImageBuilder:
"""
Builds an image from source code
"""
+ __NO_CACHE = 'no_cache'
+
def __init__(self, config, base_path, relative_source_path, tag):
"""
- TODO: pydoc
-
- :param config:
- :param base_path:
- :param relative_source_path:
- :param tag:
+ :param config: task/service config
+ :param base_path: directory containing rainbow yml
+ :param relative_source_path: source path relative to rainbow yml
+ :param tag: image tag
"""
self.base_path = base_path
self.relative_source_path = relative_source_path
@@ -51,27 +50,44 @@ class ImageBuilder:
temp_dir = self.__temp_dir()
self.__copy_source_code(temp_dir)
- self.__write_additional_files(temp_dir)
-
- # TODO: log docker output
- docker_client = docker.from_env()
- docker_client.images.build(path=temp_dir, tag=self.tag)
- docker_client.close()
+ self._write_additional_files(temp_dir)
+
+ no_cache = ''
+ if self.__NO_CACHE in self.config and self.config[self.__NO_CACHE]:
+ no_cache = '--no-cache=true'
+
+ docker_build_command = f'docker build {no_cache} --progress=plain ' + \
+ f'--tag {self.tag} {self._build_flags()} {temp_dir}'
+
+ if self._use_buildkit():
+ docker_build_command = f'DOCKER_BUILDKIT=1 {docker_build_command}'
+
+ print(docker_build_command)
+
+ docker_build_out = ''
+ try:
+ docker_build_out = subprocess.check_output(docker_build_command,
+ shell=True, stderr=subprocess.STDOUT,
+ timeout=240)
+ except subprocess.CalledProcessError as e:
+ docker_build_out = e.output
+ raise e
+ finally:
+ print('=' * 80)
+ for line in str(docker_build_out)[2:-3].split('\\n'):
+ print(line)
+ print('=' * 80)
self.__remove_dir(temp_dir)
print(f'[X] Building image: {self.tag} (Success).')
+ return docker_build_out
+
def __copy_source_code(self, temp_dir):
self.__copy_dir(os.path.join(self.base_path, self.relative_source_path), temp_dir)
- def __write_additional_files(self, temp_dir):
- # TODO: move requirements.txt related code to a parent class for python image builders.
- requirements_file_path = os.path.join(temp_dir, 'requirements.txt')
- if not os.path.exists(requirements_file_path):
- with open(requirements_file_path, 'w'):
- pass
-
+ def _write_additional_files(self, temp_dir):
for file in [self._dockerfile_path()] + self._additional_files_from_paths():
self.__copy_file(file, temp_dir)
@@ -117,6 +133,18 @@ class ImageBuilder:
"""
return []
+ def _build_flags(self):
+ """
+ Additional build flags to add to docker build command.
+ """
+ return ''
+
+ def _use_buildkit(self):
+ """
+ overwrite with True to use docker buildkit
+ """
+ return False
+
class ServiceImageBuilderMixin(object):
pass
diff --git a/rainbow/build/python.py b/rainbow/build/python.py
new file mode 100644
index 0000000..0961d2b
--- /dev/null
+++ b/rainbow/build/python.py
@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from rainbow.build.image_builder import ImageBuilder
+
+
+class BasePythonImageBuilder(ImageBuilder):
+ """
+ Base class for building python images.
+ """
+
+ __PIP_CONF = 'pip_conf'
+
+ def __init__(self, config, base_path, relative_source_path, tag):
+ super().__init__(config, base_path, relative_source_path, tag)
+
+ @staticmethod
+ def _dockerfile_path():
+ raise NotImplementedError()
+
+ def _write_additional_files(self, temp_dir):
+ requirements_file_path = os.path.join(temp_dir, 'requirements.txt')
+ if not os.path.exists(requirements_file_path):
+ with open(requirements_file_path, 'w'):
+ pass
+
+ super()._write_additional_files(temp_dir)
+
+ def _additional_files_from_filename_content_pairs(self):
+ with open(self._dockerfile_path()) as original:
+ data = original.read()
+
+ data = self.__mount_pip_conf(data)
+
+ return [('Dockerfile', data)]
+
+ def __mount_pip_conf(self, data):
+ new_data = data
+
+ if self.__PIP_CONF in self.config:
+ new_data = '# syntax = docker/dockerfile:1.0-experimental\n' + data
+ new_data = new_data.replace('{{mount}}',
+ '--mount=type=secret,id=pip_config,dst=/etc/pip.conf \\\n')
+ else:
+ new_data = new_data.replace('{{mount}} ', '')
+
+ return new_data
+
+ def _build_flags(self):
+ if self.__PIP_CONF in self.config:
+ return f'--secret id=pip_config,src={self.config[self.__PIP_CONF]}'
+ else:
+ return ''
+
+ def _use_buildkit(self):
+ if self.__PIP_CONF in self.config:
+ return True
diff --git a/rainbow/build/service/python_server/Dockerfile b/rainbow/build/service/python_server/Dockerfile
index 6119437..4d4254f 100644
--- a/rainbow/build/service/python_server/Dockerfile
+++ b/rainbow/build/service/python_server/Dockerfile
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
# Use an official Python runtime as a parent image
FROM python:3.7-slim
@@ -11,14 +29,14 @@ WORKDIR /app
# Be careful when changing this code. !
# Install any needed packages specified in python_server_requirements.txt and requirements.txt
-COPY ./python_server_requirements.txt /app
+COPY ./python_server_requirements.txt /app/
RUN pip install -r python_server_requirements.txt
-COPY ./requirements.txt /app
-RUN pip install -r requirements.txt
+COPY ./requirements.txt /app/
+RUN {{mount}} pip install -r requirements.txt
# Copy the current directory contents into the container at /app
RUN echo "Copying source code.."
-COPY . /app
+COPY . /app/
CMD python -u rainbow_python_server.py
diff --git a/rainbow/build/service/python_server/python_server.py b/rainbow/build/service/python_server/python_server.py
index 3404abf..0b2537d 100644
--- a/rainbow/build/service/python_server/python_server.py
+++ b/rainbow/build/service/python_server/python_server.py
@@ -20,10 +20,11 @@ import os
import yaml
-from rainbow.build.image.image_builder import ImageBuilder, ServiceImageBuilderMixin
+from rainbow.build.image_builder import ServiceImageBuilderMixin
+from rainbow.build.python import BasePythonImageBuilder
-class PythonServerImageBuilder(ImageBuilder, ServiceImageBuilderMixin):
+class PythonServerImageBuilder(BasePythonImageBuilder, ServiceImageBuilderMixin):
def __init__(self, config, base_path, relative_source_path, tag):
super().__init__(config, base_path, relative_source_path, tag)
@@ -40,4 +41,5 @@ class PythonServerImageBuilder(ImageBuilder, ServiceImageBuilderMixin):
]
def _additional_files_from_filename_content_pairs(self):
- return [('service.yml', yaml.safe_dump(self.config))]
+ return super()._additional_files_from_filename_content_pairs() + \
+ [('service.yml', yaml.safe_dump(self.config))]
diff --git a/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py b/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
index eb6fa83..c44e80b 100644
--- a/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
+++ b/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
@@ -67,7 +67,6 @@ class PrepareInputOperator(KubernetesPodOperator):
else:
raise ValueError(f'Unknown config type: {self.input_type}')
- # TODO: pass run_id as well as env var
run_id = context['dag_run'].run_id
print(f'run_id = {run_id}')
@@ -145,4 +144,8 @@ class KubernetesPodOperatorWithInputAndOutput(KubernetesPodOperator):
self.log.info(f'Empty input for task {self.task_split}.')
+ run_id = context['dag_run'].run_id
+ print(f'run_id = {run_id}')
+
+ self.env_vars.update({'run_id': run_id})
return super().execute(context)
diff --git a/run_tests.sh b/run_tests.sh
index 3e5cd2f..8fdae7a 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,3 +1,5 @@
#!/bin/sh
-python -m unittest
\ No newline at end of file
+export TMPDIR=/tmp
+
+python -m unittest
diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
index 63fc8fa..ecdaced 100644
--- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
+++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
import os
import threading
import time
@@ -41,23 +42,47 @@ class TestPythonServer(TestCase):
self.docker_client.close()
def test_build_python_server(self):
+ build_out = self.__test_build_python_server()
+
+ self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
+
+ def test_build_python_server_with_pip_conf(self):
+ build_out = self.__test_build_python_server(use_pip_conf=True)
+
+ self.assertTrue(
+ 'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf pip insta...' in build_out,
+ 'Incorrect pip command')
+
+ def __test_build_python_server(self, use_pip_conf=False):
base_path = os.path.join(os.path.dirname(__file__), '../../../rainbow')
- builder = PythonServerImageBuilder(config=self.config,
+
+ config = self.__create_conf('my_task')
+
+ if use_pip_conf:
+ config['pip_conf'] = os.path.join(base_path, 'pip.conf')
+
+ builder = PythonServerImageBuilder(config=config,
base_path=base_path,
relative_source_path='myserver',
tag=self.image_name)
- builder.build()
+ build_out = str(builder.build())
thread = threading.Thread(target=self.__run_container, args=[self.image_name])
thread.daemon = True
thread.start()
- time.sleep(2)
+ time.sleep(5)
+
+ print('Sending request to server')
+
+ server_response = str(urllib.request.urlopen('http://localhost:9294/myendpoint1').read())
+
+ print(f'Response from server: {server_response}')
- server_response = urllib.request.urlopen("http://localhost:9294/myendpoint1").read()
+ self.assertEqual("b'1'", server_response)
- self.assertEqual("b'1'", str(server_response))
+ return build_out
def __remove_containers(self):
print(f'Stopping containers with image: {self.image_name}')
@@ -92,6 +117,7 @@ class TestPythonServer(TestCase):
'input_type': 'my_input_type',
'input_path': 'my_input',
'output_path': '/my_output.json',
+ 'no_cache': True,
'endpoints': [
{
'endpoint': '/myendpoint1',
diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py
index 7376987..81b5cc3 100644
--- a/tests/runners/airflow/build/python/test_python_image_builder.py
+++ b/tests/runners/airflow/build/python/test_python_image_builder.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
import os
+import shutil
+import tempfile
from unittest import TestCase
import docker
@@ -24,46 +26,106 @@ from rainbow.build.image.python.python import PythonImageBuilder
class TestPythonImageBuilder(TestCase):
+ __IMAGE_NAME = 'rainbow_image'
+ __OUTPUT_PATH = '/mnt/vol1/my_output.json'
+
+ def setUp(self) -> None:
+ super().setUp()
+ os.environ['TMPDIR'] = '/tmp'
+ self.temp_dir = self.__temp_dir()
+ self.temp_airflow_dir = self.__temp_dir()
+
+ def tearDown(self) -> None:
+ super().tearDown()
+ self.__remove_dir(self.temp_dir)
+ self.__remove_dir(self.temp_airflow_dir)
def test_build(self):
- config = self.__create_conf('my_task')
+ build_out = self.__test_build()
+
+ self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
+
+ self.__test_image()
- image_name = config['image']
+ def test_build_with_pip_conf(self):
+ build_out = self.__test_build(use_pip_conf=True)
+
+ self.assertTrue(
+ 'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf pip insta...' in build_out,
+ 'Incorrect pip command')
+
+ self.__test_image()
+
+ def __test_build(self, use_pip_conf=False):
+ config = self.__create_conf('my_task')
base_path = os.path.join(os.path.dirname(__file__), '../../rainbow')
+ if use_pip_conf:
+ config['pip_conf'] = os.path.join(base_path, 'pip.conf')
+
builder = PythonImageBuilder(config=config,
base_path=base_path,
relative_source_path='helloworld',
- tag=image_name)
+ tag=self.__IMAGE_NAME)
- builder.build()
+ build_out = str(builder.build())
- # TODO: elaborate test of image, validate input/output
+ return build_out
+ def __test_image(self):
docker_client = docker.from_env()
- docker_client.images.get(image_name)
+ docker_client.images.get(self.__IMAGE_NAME)
- cmd = 'export RAINBOW_INPUT="{}" && ' + \
+ cmd = 'export RAINBOW_INPUT="{\\"x\\": 1}" && ' + \
'sh container-setup.sh && ' + \
'python hello_world.py && ' + \
- 'sh container-teardown.sh'
+ f'sh container-teardown.sh {self.__OUTPUT_PATH}'
cmds = ['/bin/bash', '-c', cmd]
- container_log = docker_client.containers.run(image_name, cmds)
+ container_log = docker_client.containers.run(self.__IMAGE_NAME,
+ cmds,
+ volumes={
+ self.temp_dir: {
+ 'bind': '/mnt/vol1',
+ 'mode': 'rw'
+ },
+ self.temp_airflow_dir: {
+ 'bind': '/airflow/xcom',
+ 'mode': 'rw'},
+ })
docker_client.close()
- self.assertEqual("b'Hello world!\\n\\n{}\\n'", str(container_log))
+ print(container_log)
- @staticmethod
- def __create_conf(task_id):
+ self.assertEqual(
+ "b\"Writing rainbow input..\\n" +
+ "Hello world!\\n\\n" +
+ "rainbow_input.json contents = {'x': 1}\\n" +
+ "Writing rainbow output..\\n\"",
+ str(container_log))
+
+ with open(os.path.join(self.temp_airflow_dir, 'return.json')) as file:
+ self.assertEqual(file.read(), '{"a": 1, "b": 2}')
+
+ def __create_conf(self, task_id):
return {
'task': task_id,
'cmd': 'foo bar',
- 'image': 'rainbow_image',
+ 'image': self.__IMAGE_NAME,
'source': 'baz',
'input_type': 'my_input_type',
'input_path': 'my_input',
- 'output_path': '/my_output.json'
+ 'no_cache': True,
+ 'output_path': self.__OUTPUT_PATH,
}
+
+ @staticmethod
+ def __temp_dir():
+ temp_dir = tempfile.mkdtemp()
+ return temp_dir
+
+ @staticmethod
+ def __remove_dir(temp_dir):
+ shutil.rmtree(temp_dir, ignore_errors=True)
diff --git a/tests/runners/airflow/build/test_build_rainbows.py b/tests/runners/airflow/build/test_build_rainbows.py
index c5d8ea7..7e01245 100644
--- a/tests/runners/airflow/build/test_build_rainbows.py
+++ b/tests/runners/airflow/build/test_build_rainbows.py
@@ -42,7 +42,7 @@ class TestBuildRainbows(TestCase):
def __remove_images(self):
for image_name in self.__image_names:
if len(self.docker_client.images.list(image_name)) > 0:
- self.docker_client.images.remove(image=image_name)
+ self.docker_client.images.remove(image=image_name, force=True)
def test_build_rainbow(self):
build_rainbows.build_rainbows(os.path.join(os.path.dirname(__file__), '../rainbow'))
diff --git a/tests/runners/airflow/rainbow/helloworld/hello_world.py b/tests/runners/airflow/rainbow/helloworld/hello_world.py
index 3eae465..95f4e73 100644
--- a/tests/runners/airflow/rainbow/helloworld/hello_world.py
+++ b/tests/runners/airflow/rainbow/helloworld/hello_world.py
@@ -16,12 +16,14 @@
# specific language governing permissions and limitations
# under the License.
import json
+import os
-print('Hello world!')
-print()
+print('Hello world!\n')
with open('/rainbow_input.json') as file:
- print(json.loads(file.readline()))
+ print(f'rainbow_input.json contents = {json.loads(file.readline())}')
-with open('/output.json', 'w') as file:
+os.makedirs('/mnt/vol1/', exist_ok=True)
+
+with open('/mnt/vol1/my_output.json', 'w') as file:
file.write(json.dumps({'a': 1, 'b': 2}))
diff --git a/tests/runners/airflow/rainbow/helloworld/hello_world.py b/tests/runners/airflow/rainbow/pip.conf
similarity index 78%
copy from tests/runners/airflow/rainbow/helloworld/hello_world.py
copy to tests/runners/airflow/rainbow/pip.conf
index 3eae465..217e5db 100644
--- a/tests/runners/airflow/rainbow/helloworld/hello_world.py
+++ b/tests/runners/airflow/rainbow/pip.conf
@@ -15,13 +15,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import json
-
-print('Hello world!')
-print()
-
-with open('/rainbow_input.json') as file:
- print(json.loads(file.readline()))
-
-with open('/output.json', 'w') as file:
- file.write(json.dumps({'a': 1, 'b': 2}))
diff --git a/tests/runners/airflow/rainbow/rainbow.yml b/tests/runners/airflow/rainbow/rainbow.yml
index 0b08a1f..77af37b 100644
--- a/tests/runners/airflow/rainbow/rainbow.yml
+++ b/tests/runners/airflow/rainbow/rainbow.yml
@@ -29,8 +29,8 @@ pipelines:
key1: val1
key2: val2
metrics:
- namespace: TestNamespace
- backends: [ 'cloudwatch' ]
+ namespace: TestNamespace
+ backends: [ 'cloudwatch' ]
tasks:
- task: my_static_input_task
type: python
@@ -42,7 +42,7 @@ pipelines:
env2: "b"
input_type: static
input_path: '[ { "foo": "bar" }, { "foo": "baz" } ]'
- output_path: /output.json
+ output_path: /mnt/vol1/my_output.json
cmd: python -u hello_world.py
- task: my_parallelized_static_input_task
type: python
@@ -55,7 +55,7 @@ pipelines:
input_path: '[ { "foo": "bar" }, { "foo": "baz" } ]'
split_input: True
executors: 2
- cmd: python -u helloworld.py
+ cmd: python -u hello_world.py
- task: my_task_output_input_task
type: python
description: task with input from other task's output