You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@liminal.apache.org by av...@apache.org on 2020/12/24 12:10:26 UTC
[incubator-liminal] branch master updated: Configurable python
version
This is an automated email from the ASF dual-hosted git repository.
aviemzur pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new 0b961ea Configurable python version
0b961ea is described below
commit 0b961ea554e72f19e89aaace871f722fc3df54aa
Author: roei <ro...@naturalint.com>
AuthorDate: Thu Dec 24 14:10:18 2020 +0200
Configurable python version
---
liminal/build/image/python/Dockerfile | 2 +-
liminal/build/python.py | 43 +++++++++++++++++++++-
liminal/build/service/python_server/Dockerfile | 2 +-
.../python/test_python_server_image_builder.py | 16 +++++---
.../build/python/test_python_image_builder.py | 18 +++++++--
5 files changed, 69 insertions(+), 12 deletions(-)
diff --git a/liminal/build/image/python/Dockerfile b/liminal/build/image/python/Dockerfile
index 4a91690..67d8462 100644
--- a/liminal/build/image/python/Dockerfile
+++ b/liminal/build/image/python/Dockerfile
@@ -17,7 +17,7 @@
# under the License.
# Use an official Python runtime as a parent image
-FROM python:3.7-slim
+FROM {{python}}
# Install aptitude build-essential
#RUN apt-get install -y --reinstall build-essential
diff --git a/liminal/build/python.py b/liminal/build/python.py
index 07c6155..e42b63b 100644
--- a/liminal/build/python.py
+++ b/liminal/build/python.py
@@ -21,15 +21,50 @@ import os
from liminal.build.image_builder import ImageBuilder
+class PythonImageVersions:
+ """
+ Handles the python versions for python images.
+ """
+
+ @property
+ def default_version(self):
+ return '3.7'
+
+ @property
+ def supported_versions(self):
+ return '3.6', '3.7', '3.8', '3.9'
+
+ def get_image_name(self, python_version):
+ """
+ :param python_version: The python version that would be installed in
+ the docker image. For example '3.8', '3.8.1' etc.
+ :type python_version: str
+ :return: The name of the base (slim) python image
+ :rtype: str
+ """
+ if not python_version:
+ python_version = self.default_version
+ else:
+ python_version = str(python_version)
+ if python_version[:3] not in self.supported_versions:
+ raise ValueError(f'liminal supports the following python versions: '
+ f'{self.supported_versions} but {python_version} '
+ f'were passed')
+ return f'python:{python_version}-slim'
+
+
class BasePythonImageBuilder(ImageBuilder):
"""
Base class for building python images.
"""
__PIP_CONF = 'pip_conf'
+ __PYTHON_VERSION = 'python_version'
- def __init__(self, config, base_path, relative_source_path, tag):
+ def __init__(self, config, base_path, relative_source_path, tag,
+ base_image=PythonImageVersions()):
super().__init__(config, base_path, relative_source_path, tag)
+ self._base_image = base_image
@staticmethod
def _dockerfile_path():
@@ -48,6 +83,7 @@ class BasePythonImageBuilder(ImageBuilder):
data = original.read()
data = self.__mount_pip_conf(data)
+ data = self.__add_python_base_version(data)
return [('Dockerfile', data)]
@@ -63,6 +99,11 @@ class BasePythonImageBuilder(ImageBuilder):
return new_data
+ def __add_python_base_version(self, data):
+ python_version = self.config.get(self.__PYTHON_VERSION)
+ base_image = self._base_image.get_image_name(python_version)
+ return data.replace('{{python}}', base_image)
+
def _build_flags(self):
if self.__PIP_CONF in self.config:
return f'--secret id=pip_config,src={self.config[self.__PIP_CONF]}'
diff --git a/liminal/build/service/python_server/Dockerfile b/liminal/build/service/python_server/Dockerfile
index 96c2e4a..302c7b9 100644
--- a/liminal/build/service/python_server/Dockerfile
+++ b/liminal/build/service/python_server/Dockerfile
@@ -17,7 +17,7 @@
# under the License.
# Use an official Python runtime as a parent image
-FROM python:3.7-slim
+FROM {{python}}
# Install aptitude build-essential
#RUN apt-get install -y --reinstall build-essential
diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
index 0923009..18215d3 100644
--- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
+++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
@@ -27,7 +27,7 @@ from unittest import TestCase
import docker
from liminal.build.service.python_server.python_server import PythonServerImageBuilder
-
+from liminal.build.python import PythonImageVersions
class TestPythonServer(TestCase):
@@ -43,9 +43,11 @@ class TestPythonServer(TestCase):
self.docker_client.close()
def test_build_python_server(self):
- build_out = self.__test_build_python_server()
-
- self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
+ versions = [None] + list(PythonImageVersions().supported_versions)
+ for version in versions:
+ build_out = self.__test_build_python_server(python_version=version)
+ self.assertTrue('RUN pip install -r requirements.txt' in build_out,
+ 'Incorrect pip command')
def test_build_python_server_with_pip_conf(self):
build_out = self.__test_build_python_server(use_pip_conf=True)
@@ -54,7 +56,8 @@ class TestPythonServer(TestCase):
'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf pip install' in build_out,
'Incorrect pip command')
- def __test_build_python_server(self, use_pip_conf=False):
+ def __test_build_python_server(self, use_pip_conf=False,
+ python_version=None):
base_path = os.path.join(os.path.dirname(__file__), '../../../liminal')
config = self.__create_conf('my_task')
@@ -62,6 +65,9 @@ class TestPythonServer(TestCase):
if use_pip_conf:
config['pip_conf'] = os.path.join(base_path, 'pip.conf')
+ if python_version:
+ config['python_version'] = python_version
+
builder = PythonServerImageBuilder(config=config,
base_path=base_path,
relative_source_path='myserver',
diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py
index 668af02..56667bd 100644
--- a/tests/runners/airflow/build/python/test_python_image_builder.py
+++ b/tests/runners/airflow/build/python/test_python_image_builder.py
@@ -23,6 +23,7 @@ from unittest import TestCase
import docker
from liminal.build.image.python.python import PythonImageBuilder
+from liminal.build.python import PythonImageVersions
class TestPythonImageBuilder(TestCase):
@@ -40,9 +41,11 @@ class TestPythonImageBuilder(TestCase):
self.__remove_dir(self.temp_airflow_dir)
def test_build(self):
- build_out = self.__test_build()
-
- self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
+ for python_version in [None ,
+ PythonImageVersions().supported_versions[0]]:
+ build_out = self.__test_build(python_version=python_version)
+ self.assertTrue('RUN pip install -r requirements.txt' in build_out,
+ 'Incorrect pip command')
self.__test_image()
@@ -55,7 +58,11 @@ class TestPythonImageBuilder(TestCase):
self.__test_image()
- def __test_build(self, use_pip_conf=False):
+ def test_with_unsupported_python_version(self):
+ with self.assertRaises(ValueError):
+ self.__test_build(python_version='3.5.2')
+
+ def __test_build(self, use_pip_conf=False, python_version=None):
config = self.__create_conf('my_task')
base_path = os.path.join(os.path.dirname(__file__), '../../liminal')
@@ -63,6 +70,9 @@ class TestPythonImageBuilder(TestCase):
if use_pip_conf:
config['pip_conf'] = os.path.join(base_path, 'pip.conf')
+ if python_version:
+ config['python_version'] = python_version
+
builder = PythonImageBuilder(config=config,
base_path=base_path,
relative_source_path='write_inputs',