You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@liminal.apache.org by av...@apache.org on 2021/05/18 11:15:27 UTC
[incubator-liminal] branch master updated: Support AWS MWAA
This is an automated email from the ASF dual-hosted git repository.
aviemzur pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new 59216d1 Support AWS MWAA
59216d1 is described below
commit 59216d1cef2ea78d6fc329167ed1d1793f8e71b0
Author: Lidor Ettinger <li...@gmail.com>
AuthorDate: Tue May 18 14:15:20 2021 +0300
Support AWS MWAA
---
MANIFEST.in | 4 ++--
liminal/core/environment.py | 3 +++
liminal/runners/airflow/tasks/python.py | 2 ++
requirements.txt | 1 -
4 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 14ca7de..662861a 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -17,5 +17,5 @@
# under the License.
-include scripts/* requirements.txt
-recursive-include liminal/build/ *
+include scripts/* requirements.txt requirements-airflow.txt
+recursive-include liminal/build/ *
\ No newline at end of file
diff --git a/liminal/core/environment.py b/liminal/core/environment.py
index ab6adaf..e8f9ad4 100644
--- a/liminal/core/environment.py
+++ b/liminal/core/environment.py
@@ -24,10 +24,13 @@ DEFAULT_DAGS_ZIP_NAME = 'liminal.zip'
DEFAULT_LIMINAL_HOME = os.path.expanduser('~/liminal_home')
DEFAULT_PIPELINES_SUBDIR = "pipelines"
LIMINAL_HOME_PARAM_NAME = "LIMINAL_HOME"
+LIMINAL_HOME_CLOUD_PARAM_NAME = "AIRFLOW__CORE__LIMINAL_HOME"
LIMINAL_VERSION_PARAM_NAME = 'LIMINAL_VERSION'
def get_liminal_home():
+ if LIMINAL_HOME_CLOUD_PARAM_NAME in os.environ:
+ return os.environ.get(LIMINAL_HOME_CLOUD_PARAM_NAME)
if not os.environ.get(LIMINAL_HOME_PARAM_NAME):
logging.info("no environment parameter called LIMINAL_HOME detected")
logging.info(f"registering {DEFAULT_LIMINAL_HOME} as the LIMINAL_HOME directory")
diff --git a/liminal/runners/airflow/tasks/python.py b/liminal/runners/airflow/tasks/python.py
index e359a7c..5030b7d 100644
--- a/liminal/runners/airflow/tasks/python.py
+++ b/liminal/runners/airflow/tasks/python.py
@@ -154,6 +154,8 @@ class PythonTask(task.Task):
'resources': self.resources,
'dag': self.dag,
'volumes': self.volumes,
+ 'config_file': get_variable('kube_config_path', default_val='~/.kube/config'),
+ 'cluster_context': get_variable('cluster_context', default_val='None'),
'volume_mounts': [
VolumeMount(mount['volume'],
mount['path'],
diff --git a/requirements.txt b/requirements.txt
index 96b1dbb..c550659 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,7 +29,6 @@ termcolor~=1.1.0
docker-pycreds==0.4.0
typing==3.7.4.1
GitPython==3.1.11
-ddtrace==0.37.0
moto==1.3.14
diskcache==3.1.1
croniter==0.3.31