[incubator-liminal] branch official_docker updated: support deploying local/unofficial vesions of liminal inside airflow

[as...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

assafpinhasi pushed a commit to branch official_docker
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git


The following commit(s) were added to refs/heads/official_docker by this push:
     new f60ec20  support deploying local/unofficial vesions of liminal inside airflow
f60ec20 is described below

commit f60ec2008ea9e68164d64106c9b6285105210c3a
Author: Assaf Pinhasi <as...@Assafs-MacBook-Pro.local>
AuthorDate: Sat Oct 31 23:45:48 2020 +0200

    support deploying local/unofficial vesions of liminal inside airflow
---
 liminal/core/environment.py | 19 +++++++++++++++++++
 scripts/Dockerfile-airflow  | 10 ++++++++--
 scripts/liminal             | 14 ++++++++++++--
 setup.py                    |  2 +-
 4 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/liminal/core/environment.py b/liminal/core/environment.py
index f335ce9..5f81b09 100644
--- a/liminal/core/environment.py
+++ b/liminal/core/environment.py
@@ -17,11 +17,13 @@
 # under the License.
 
 import os
+import subprocess
 
 DEFAULT_DAGS_ZIP_NAME = 'liminal.zip'
 DEFAULT_LIMINAL_HOME = os.path.expanduser('~/liminal_home')
 DEFAULT_PIPELINES_SUBDIR = "pipelines"
 LIMINAL_HOME_PARAM_NAME = "LIMINAL_HOME"
+LIMINAL_VERSION_PARAM_NAME = 'LIMINAL_VERSION'
 
 
 def get_liminal_home():
@@ -36,3 +38,20 @@ def get_dags_dir():
     # if we are inside airflow, we will take it from the configured dags folder
     base_dir = os.environ.get("AIRFLOW__CORE__DAGS_FOLDER", get_liminal_home())
     return os.path.join(base_dir, DEFAULT_PIPELINES_SUBDIR)
+
+
+def get_liminal_version():
+    result = os.environ.get(LIMINAL_VERSION_PARAM_NAME, None)
+    if not result:
+        output = subprocess.run(['pip freeze | grep \'apache-liminal\''], capture_output=True, env=os.environ, shell=True)
+        pip_res = output.stdout.decode('UTF-8').strip()
+        value = None
+        if not pip_res:
+            value = 'apache-liminal'
+        if ' @ ' in pip_res:
+            value = pip_res[pip_res.index(' @ ')+3:]
+        else:
+            value= pip_res
+        print(f'LIMINAL_VERSION not set. Setting it to currently installed version: {value}')
+        os.environ[LIMINAL_VERSION_PARAM_NAME] = value
+    return os.environ.get(LIMINAL_VERSION_PARAM_NAME, 'apache-liminal')
diff --git a/scripts/Dockerfile-airflow b/scripts/Dockerfile-airflow
index 32b9899..c2095d6 100644
--- a/scripts/Dockerfile-airflow
+++ b/scripts/Dockerfile-airflow
@@ -5,7 +5,13 @@ ARG DAG_FOLDER='/opt/airflow/dags/'
 ADD scripts/* ${DAG_FOLDER}
 ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER}
 
-RUN pip install -r /opt/airflow/dags/requirements-airflow.txt --user
-RUN pip install --user --no-deps ${LIMINAL_VERSION}
+USER root
+
+RUN apt-get update && apt-get install -y --no-install-recommends git
+
+USER airflow
+
+RUN pip --disable-pip-version-check install -r /opt/airflow/dags/requirements-airflow.txt --user
+RUN pip --disable-pip-version-check install --user --no-deps ${LIMINAL_VERSION}
 
 
diff --git a/scripts/liminal b/scripts/liminal
index 048b31b..76eb0e3 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -65,9 +65,19 @@ def deploy_liminal_core_internal(clean):
     os.makedirs(environment.get_dags_dir(), exist_ok=True)
     dag_target_file = os.path.join(environment.get_liminal_home(), 'liminal_dags.py')
     shutil.copyfile(dags_path, dag_target_file)
+    #initialize the env. variable which indicates to the docke compose which liminal to install in airflow docker
+    liminal_version = environment.get_liminal_version()
+    # if liminal is installed from local file - teh developer needs to put it in the /scipts folder, in which case
+    # it will end up inside the container during build
+    if liminal_version.find("file://")>-1:
+        local_file_name = os.path.basename(liminal_version)
+        full_path = os.path.join('/opt/airflow/dags',local_file_name)
+        print(f'Liminal was installed locally, changing the LIMINAL_VERSION parameter to {full_path}')
+        os.environ[environment.LIMINAL_VERSION_PARAM_NAME]= full_path
     if clean:
-        docker_compose_command('build', ['no-cache'])
-        docker_compose_command('run',['webserver','initdb'])
+        docker_compose_command('down', ['--rmi', 'local'])
+        docker_compose_command('build', ['--no-cache'])
+        docker_compose_command('run', ['webserver','initdb'])
 
 
 def docker_compose_command(command_name, args):
diff --git a/setup.py b/setup.py
index 7e9effc..771f602 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ with open('requirements.txt') as f:
 
 setuptools.setup(
     name="apache-liminal",
-    version=os.environ["LIMINAL_BUILD_VERSION"],
+    version=os.environ.get("LIMINAL_BUILD_VERSION", os.environ.get('LIMINAL_VERSION', None)),
     author="dev@liminal.apache.org",
     description="A package for authoring and deploying machine learning workflows",
     long_description=long_description,