[incubator-liminal] branch master updated: fix install from code (#68)

[as...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

assafpinhasi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git


The following commit(s) were added to refs/heads/master by this push:
     new 5042784  fix install from code (#68)
5042784 is described below

commit 5042784050d2fe09e206979cb2673a1efab7428f
Author: Aviem Zur <av...@gmail.com>
AuthorDate: Tue Oct 26 04:38:48 2021 -0700

    fix install from code (#68)
---
 install.sh                  | 15 +++------------
 liminal/core/environment.py |  9 +++------
 scripts/Dockerfile-airflow  |  3 ++-
 scripts/liminal             | 12 ++++++------
 4 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/install.sh b/install.sh
index ff54bf6..8bd5903 100755
--- a/install.sh
+++ b/install.sh
@@ -19,27 +19,18 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-if ! command -v liminal &> /dev/null
-then
-  liminal stop
-fi
-
 yes | pip uninstall apache-liminal
 
 cd "$DIR" || exit
 
 rm -rf build
 rm -rf dist
+rm "${LIMINAL_HOME:-${HOME}/liminal_home}"/*.whl
 
 python setup.py sdist bdist_wheel
 
-rm scripts/*.whl
-
-cp dist/*.whl scripts
-
-ver=$(ls scripts/*.whl)
-export LIMINAL_VERSION="apache-liminal @ file://$ver"
+pip install dist/*.whl
 
-pip install scripts/*.whl
+cp dist/*.whl "${LIMINAL_HOME:-${HOME}/liminal_home}"
 
 cd - || exit
diff --git a/liminal/core/environment.py b/liminal/core/environment.py
index e8f9ad4..636994c 100644
--- a/liminal/core/environment.py
+++ b/liminal/core/environment.py
@@ -50,13 +50,10 @@ def get_liminal_version():
         output = subprocess.run(['pip freeze | grep \'apache-liminal\''], capture_output=True,
                                 env=os.environ, shell=True)
         pip_res = output.stdout.decode('UTF-8').strip()
-        scripts_dir = os.path.join(
-            os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
-            'scripts'
-        )
-        whl_files = [file for file in os.listdir(scripts_dir) if file.endswith(".whl")]
+        liminal_home = get_liminal_home()
+        whl_files = [file for file in os.listdir(liminal_home) if file.endswith(".whl")]
         if whl_files:
-            value = 'file://' + os.path.join(scripts_dir, whl_files[0])
+            value = 'file://' + os.path.join(liminal_home, whl_files[0])
         elif ' @ ' in pip_res:
             value = pip_res[pip_res.index(' @ ') + 3:]
         else:
diff --git a/scripts/Dockerfile-airflow b/scripts/Dockerfile-airflow
index 872310e..284b1b9 100644
--- a/scripts/Dockerfile-airflow
+++ b/scripts/Dockerfile-airflow
@@ -21,6 +21,7 @@ FROM apache/airflow:1.10.12-python3.6
 ARG LIMINAL_VERSION='apache-liminal'
 ARG DAG_FOLDER='/opt/airflow/dags/'
 ADD scripts/* ${DAG_FOLDER}
+ADD *.whl ${DAG_FOLDER}
 
 RUN ls -ls ${DAG_FOLDER}
 ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER}
@@ -28,4 +29,4 @@ ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER}
 USER airflow
 
 RUN pip --disable-pip-version-check install -r /opt/airflow/dags/requirements-airflow.txt --user
-RUN pip --disable-pip-version-check install --user --no-deps ${LIMINAL_VERSION}
\ No newline at end of file
+RUN pip --disable-pip-version-check install --user --no-deps ${LIMINAL_VERSION}
diff --git a/scripts/liminal b/scripts/liminal
index a1c1760..397c02d 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -25,13 +25,12 @@ import subprocess
 import sys
 
 import click
-import yaml
 
 import scripts
 from liminal.build import liminal_apps_builder
 from liminal.core import environment
-from liminal.core.util import files_util
 from liminal.core.config.config import ConfigUtil
+from liminal.core.util import files_util
 from liminal.kubernetes import volume_util
 from liminal.logging.logging_setup import logging_initialization
 from liminal.runners.airflow import dag
@@ -89,10 +88,11 @@ def deploy_liminal_core_internal(liminal_home, clean):
     if liminal_version.find("file://") > -1:
         local_file_name = os.path.basename(liminal_version)
         full_path = os.path.join('/opt/airflow/dags', local_file_name)
-        logging.info(
-            f'Liminal was installed locally, changing the LIMINAL_VERSION parameter to {full_path}')
+        logging.info(f'Liminal was installed locally, setting LIMINAL_VERSION to {full_path}')
         os.environ[environment.LIMINAL_VERSION_PARAM_NAME] = full_path
-        shutil.copyfile(liminal_version[7:], os.path.join(liminal_home, local_file_name))
+        _, project_dir = get_docker_compose_paths()
+        shutil.copyfile(os.path.join(liminal_home, local_file_name),
+                        os.path.join(project_dir, local_file_name))
     if clean:
         docker_compose_command('down', ['--remove-orphans', '--rmi', 'local'])
         docker_compose_command('build', ['--no-cache'])
@@ -112,7 +112,7 @@ def docker_compose_command(command_name, args):
         f'{project_dir} {command_name} {concated_args}'
     ]
     logging.info(run_command[0])
-    if ('tail' in str(args) or '-d' in str(args) or command_name in ['ps', 'down']):
+    if 'tail' in str(args) or '-d' in str(args) or command_name in ['ps', 'down']:
         detached_mode = True
 
     if detached_mode: