You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@liminal.apache.org by av...@apache.org on 2022/01/19 08:40:16 UTC
[incubator-liminal] branch master updated: [LIMINAL-51] Allow user extensions of liminal abstractions
This is an automated email from the ASF dual-hosted git repository.
aviemzur pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new 6be6e6a [LIMINAL-51] Allow user extensions of liminal abstractions
6be6e6a is described below
commit 6be6e6ac8216267b4aeb18cace43a019d2005003
Author: Zion Rubin <zi...@naturalint.com>
AuthorDate: Wed Jan 19 10:40:10 2022 +0200
[LIMINAL-51] Allow user extensions of liminal abstractions
---
.gitignore | 2 +-
docs/liminal/extensibility.md | 63 ++++++++++++++++++++++
{liminal => examples/extensibility}/__init__.py | 0
.../extensibility/executors}/__init__.py | 0
.../extensibility/executors/custom_executor.py | 25 ++++-----
.../extensibility/images}/__init__.py | 0
.../extensibility/images/custom_image.py | 23 +++-----
.../images/custom_image_builder/Dockerfile | 5 ++
.../images/custom_image_builder}/__init__.py | 0
.gitignore => examples/extensibility/liminal.yml | 34 ++++++------
.../extensibility/tasks}/__init__.py | 0
.../extensibility/tasks/custom_task.py | 25 ++++-----
install.sh | 2 +
liminal/__init__.py | 3 ++
liminal/build/liminal_apps_builder.py | 9 +---
liminal/core/util/extensible.py | 53 ++++++++++++++++++
.../runners/airflow/dag/liminal_register_dags.py | 14 ++---
.gitignore => liminal/settings.py | 30 +++++------
scripts/docker-compose.yml | 1 +
scripts/liminal | 11 +++-
20 files changed, 204 insertions(+), 96 deletions(-)
diff --git a/.gitignore b/.gitignore
index dfaa502..5b59b1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+env
.idea
bin
include
diff --git a/docs/liminal/extensibility.md b/docs/liminal/extensibility.md
new file mode 100644
index 0000000..76a67fc
--- /dev/null
+++ b/docs/liminal/extensibility.md
@@ -0,0 +1,63 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Extensibility
+
+The extensibility designed to have an easy way for the users to add their tasks, executors, image
+builders and extend the library so that it fits the level of abstraction that suits the user
+environment.
+
+## Location
+
+Tasks folder: `{LIMINAL_HOME}/plugins/tasks`
+
+Executors folder: `{LIMINAL_HOME}/plugins/executors`
+
+Image builders folder: `{LIMINAL_HOME}/plugins/images`
+
+## Example
+
+### Prerequisites
+
+Apache Liminal
+
+### Guide
+
+Check out the examples for each one of the extensible item
+in [examples/extensibility](../../examples/extensibility)
+
+Copy the extensible items to the plugin location:
+
+```shell
+cp -r ../../examples/extensibility/executors/* $LIMINAL_HOME/liminal/plugins/executors/
+```
+
+```shell
+cp -r ../../examples/extensibility/tasks/* $LIMINAL_HOME/liminal/plugins/tasks/
+```
+
+```shell
+cp -r ../../examples/extensibility/images/* $LIMINAL_HOME/liminal/plugins/images/
+```
+
+```shell
+liminal build .
+liminal deploy --clean
+liminal start
+```
diff --git a/liminal/__init__.py b/examples/extensibility/__init__.py
similarity index 100%
copy from liminal/__init__.py
copy to examples/extensibility/__init__.py
diff --git a/liminal/__init__.py b/examples/extensibility/executors/__init__.py
similarity index 100%
copy from liminal/__init__.py
copy to examples/extensibility/executors/__init__.py
diff --git a/.gitignore b/examples/extensibility/executors/custom_executor.py
similarity index 70%
copy from .gitignore
copy to examples/extensibility/executors/custom_executor.py
index dfaa502..d2a7c00 100644
--- a/.gitignore
+++ b/examples/extensibility/executors/custom_executor.py
@@ -15,19 +15,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from plugins.tasks import custom_task
-.idea
-bin
-include
-lib
-venv
-/build
-.Python
-*.pyc
-pip-selfcheck.json
-.DS_Store
-/build
-apache_liminal.egg-info
-scripts/*.tar.gz
-scripts/*.whl
-dist
+from liminal.runners.airflow.model import executor
+
+
+class CustomExecutor(executor.Executor):
+ supported_task_types = [custom_task.CustomTask]
+
+ def _apply_executor_task_to_dag(self, **kwargs):
+ print("Hello from custom executor")
+ return kwargs['task'].custom_task_logic()
diff --git a/liminal/__init__.py b/examples/extensibility/images/__init__.py
similarity index 100%
copy from liminal/__init__.py
copy to examples/extensibility/images/__init__.py
diff --git a/install.sh b/examples/extensibility/images/custom_image.py
old mode 100755
new mode 100644
similarity index 67%
copy from install.sh
copy to examples/extensibility/images/custom_image.py
index 8bd5903..01df58c
--- a/install.sh
+++ b/examples/extensibility/images/custom_image.py
@@ -1,4 +1,3 @@
-#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,21 +15,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import os
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+from liminal.build.image_builder import ImageBuilder
-yes | pip uninstall apache-liminal
-cd "$DIR" || exit
+class CustomImageBuilder(ImageBuilder):
+ def __init__(self, config, base_path, relative_source_path, tag):
+ super().__init__(config, base_path, relative_source_path, tag)
-rm -rf build
-rm -rf dist
-rm "${LIMINAL_HOME:-${HOME}/liminal_home}"/*.whl
-
-python setup.py sdist bdist_wheel
-
-pip install dist/*.whl
-
-cp dist/*.whl "${LIMINAL_HOME:-${HOME}/liminal_home}"
-
-cd - || exit
+ @staticmethod
+ def _dockerfile_path():
+ return os.path.join(os.path.dirname(__file__), 'custom_image_builder/Dockerfile')
diff --git a/liminal/__init__.py b/examples/extensibility/images/custom_image_builder/Dockerfile
similarity index 94%
copy from liminal/__init__.py
copy to examples/extensibility/images/custom_image_builder/Dockerfile
index 217e5db..0d4063b 100644
--- a/liminal/__init__.py
+++ b/examples/extensibility/images/custom_image_builder/Dockerfile
@@ -15,3 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+FROM alpine:3.4
+
+WORKDIR /app
+
+COPY . /app/
diff --git a/liminal/__init__.py b/examples/extensibility/images/custom_image_builder/__init__.py
similarity index 100%
copy from liminal/__init__.py
copy to examples/extensibility/images/custom_image_builder/__init__.py
diff --git a/.gitignore b/examples/extensibility/liminal.yml
similarity index 64%
copy from .gitignore
copy to examples/extensibility/liminal.yml
index dfaa502..3a44df6 100644
--- a/.gitignore
+++ b/examples/extensibility/liminal.yml
@@ -15,19 +15,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-.idea
-bin
-include
-lib
-venv
-/build
-.Python
-*.pyc
-pip-selfcheck.json
-.DS_Store
-/build
-apache_liminal.egg-info
-scripts/*.tar.gz
-scripts/*.whl
-dist
+---
+name: Extensability
+images:
+ - image: custom_image_example
+ type: custom_image
+executors:
+ - executor: my_custom_exec
+ type: custom_executor
+pipelines:
+ - pipeline: getting_started_pipeline_extensibility
+ owner: Bosco Albert Baracus
+ start_date: 1970-01-01
+ timeout_minutes: 10
+ schedule: 0 * 1 * *
+ tasks:
+ - task: custom_task_example
+ type: custom_task
+ executor: my_custom_exec
diff --git a/liminal/__init__.py b/examples/extensibility/tasks/__init__.py
similarity index 100%
copy from liminal/__init__.py
copy to examples/extensibility/tasks/__init__.py
diff --git a/install.sh b/examples/extensibility/tasks/custom_task.py
old mode 100755
new mode 100644
similarity index 66%
copy from install.sh
copy to examples/extensibility/tasks/custom_task.py
index 8bd5903..112ce09
--- a/install.sh
+++ b/examples/extensibility/tasks/custom_task.py
@@ -1,4 +1,3 @@
-#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,21 +15,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from airflow.operators.bash import BashOperator
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+from liminal.runners.airflow.model import task
-yes | pip uninstall apache-liminal
-cd "$DIR" || exit
+class CustomTask(task.Task):
+ def custom_task_logic(self):
+ hello_world = BashOperator(
+ task_id='hello_world',
+ bash_command='echo "hello from liminal custom task"',
+ )
-rm -rf build
-rm -rf dist
-rm "${LIMINAL_HOME:-${HOME}/liminal_home}"/*.whl
+ if self.parent:
+ self.parent.set_downstream(hello_world)
-python setup.py sdist bdist_wheel
-
-pip install dist/*.whl
-
-cp dist/*.whl "${LIMINAL_HOME:-${HOME}/liminal_home}"
-
-cd - || exit
+ return hello_world
diff --git a/install.sh b/install.sh
index 8bd5903..1ce7102 100755
--- a/install.sh
+++ b/install.sh
@@ -31,6 +31,8 @@ python setup.py sdist bdist_wheel
pip install dist/*.whl
+mkdir -p "${LIMINAL_HOME:-${HOME}/liminal_home}"
+
cp dist/*.whl "${LIMINAL_HOME:-${HOME}/liminal_home}"
cd - || exit
diff --git a/liminal/__init__.py b/liminal/__init__.py
index 217e5db..fadce5d 100644
--- a/liminal/__init__.py
+++ b/liminal/__init__.py
@@ -15,3 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from liminal import settings
+
+settings.initialize()
diff --git a/liminal/build/liminal_apps_builder.py b/liminal/build/liminal_apps_builder.py
index a31390f..f5dc21f 100644
--- a/liminal/build/liminal_apps_builder.py
+++ b/liminal/build/liminal_apps_builder.py
@@ -19,9 +19,8 @@
import logging
import os
-from liminal.build.image_builder import ImageBuilder
from liminal.core.config.config import ConfigUtil
-from liminal.core.util import class_util, files_util
+from liminal.core.util import class_util, extensible, files_util
def build_liminal_apps(path):
@@ -64,11 +63,7 @@ def __get_image_builder_class(task_type):
logging.info(f'Loading image builder implementations..')
-# TODO: add configuration for user image builders package
-image_builders_package = 'liminal.build.image'
-# user_image_builders_package = 'TODO: user_image_builders_package'
-
-image_builder_types = class_util.find_subclasses_in_packages([image_builders_package], ImageBuilder)
+image_builder_types = extensible.load_image_builders()
logging.info(f'Finished loading image builder implementations: {image_builder_types}')
logging.info(f'Loading service image builder implementations..')
diff --git a/liminal/core/util/extensible.py b/liminal/core/util/extensible.py
new file mode 100644
index 0000000..dc39839
--- /dev/null
+++ b/liminal/core/util/extensible.py
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+
+from liminal.build.image_builder import ImageBuilder
+from liminal.core.util import class_util
+from liminal.runners.airflow.model import executor
+from liminal.runners.airflow.model.task import Task
+
+__PLUGINS = 'plugins'
+
+
+def __generate_extra_paths(plugin_type, extra_paths):
+ return (extra_paths or []) + ([f'{__PLUGINS}.{plugin_type}'] if f'{__PLUGINS}.{plugin_type}' in sys.path else [])
+
+
+def load_executors(extra_paths=None):
+ """
+ Load all Executor extensions
+ """
+ package_paths = ['liminal.runners.airflow.executors'] + __generate_extra_paths('executors', extra_paths)
+ return class_util.find_subclasses_in_packages(package_paths, executor.Executor)
+
+
+def load_tasks(extra_paths=None):
+ """
+ Load all Task extensions
+ """
+ package_paths = ['liminal.runners.airflow.tasks'] + __generate_extra_paths('tasks', extra_paths)
+ return class_util.find_subclasses_in_packages(package_paths, Task)
+
+
+def load_image_builders(extra_paths=None):
+ """
+ Load all ImageBuilder extensions
+ """
+ package_paths = ['liminal.build.image'] + __generate_extra_paths('images', extra_paths)
+ return class_util.find_subclasses_in_packages(package_paths, ImageBuilder)
diff --git a/liminal/runners/airflow/dag/liminal_register_dags.py b/liminal/runners/airflow/dag/liminal_register_dags.py
index b5e921e..eac03d6 100644
--- a/liminal/runners/airflow/dag/liminal_register_dags.py
+++ b/liminal/runners/airflow/dag/liminal_register_dags.py
@@ -23,10 +23,8 @@ from datetime import datetime, timedelta
from airflow import DAG
from liminal.core.config.config import ConfigUtil
-from liminal.core.util import class_util
+from liminal.core.util import extensible
from liminal.runners.airflow.executors import airflow
-from liminal.runners.airflow.model import executor as liminal_executor
-from liminal.runners.airflow.model.task import Task
__DEPENDS_ON_PAST = 'depends_on_past'
@@ -158,17 +156,11 @@ def __default_args(pipeline):
logging.info(f'Loading task implementations..')
-# TODO: add configuration for user tasks package
-impl_packages = 'liminal.runners.airflow.tasks'
-user_task_package = 'TODO: user_tasks_package'
-
-tasks_by_liminal_name = class_util.find_subclasses_in_packages([impl_packages], Task)
+tasks_by_liminal_name = extensible.load_tasks()
logging.info(f'Finished loading task implementations: {tasks_by_liminal_name.keys()}')
-executors_by_liminal_name = class_util.find_subclasses_in_packages(
- ['liminal.runners.airflow.executors'], liminal_executor.Executor
-)
+executors_by_liminal_name = extensible.load_executors()
logging.info(f'Finished loading executor implementations: {executors_by_liminal_name.keys()}')
diff --git a/.gitignore b/liminal/settings.py
similarity index 71%
copy from .gitignore
copy to liminal/settings.py
index dfaa502..e8c06dc 100644
--- a/.gitignore
+++ b/liminal/settings.py
@@ -15,19 +15,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import os
+import sys
-.idea
-bin
-include
-lib
-venv
-/build
-.Python
-*.pyc
-pip-selfcheck.json
-.DS_Store
-/build
-apache_liminal.egg-info
-scripts/*.tar.gz
-scripts/*.whl
-dist
+from liminal.core.environment import get_liminal_home
+
+LIMINAL_HOME = get_liminal_home()
+
+
+def prepare_syspath():
+ plugins = os.path.join(LIMINAL_HOME, 'liminal')
+ sys.path.append(plugins)
+
+
+def initialize():
+ # making sure all extensible modules are in root path
+ prepare_syspath()
diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index b987537..d86ef2e 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -38,6 +38,7 @@ x-airflow-common: &airflow-common
AIRFLOW_CONN_METADATA_DB: postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
AIRFLOW_VAR__METADATA_DB_SCHEMA: airflow
volumes:
+ - ${LIMINAL_HOME}/liminal:/opt/airflow/dags/liminal/
- ${LIMINAL_HOME}:/opt/airflow/dags
- ${LIMINAL_HOME}/logs:/opt/airflow/logs
- ${HOME}/.kube:/home/airflow/kube
diff --git a/scripts/liminal b/scripts/liminal
index a56643f..8ec663f 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -27,6 +27,7 @@ import sys
import click
import scripts
+from liminal import settings
from liminal.build import liminal_apps_builder
from liminal.core import environment
from liminal.core.config.config import ConfigUtil
@@ -36,6 +37,7 @@ from liminal.logging.logging_setup import logging_initialization
from liminal.runners.airflow import dag
logging_initialization()
+settings.initialize()
try:
import importlib.resources as pkg_resources
@@ -65,7 +67,7 @@ def build(path):
if docker_is_running():
liminal_apps_builder.build_liminal_apps(path)
configs = ConfigUtil(path).safe_load(is_render_variables=True,
- soft_merge=True)
+ soft_merge=True)
for config in configs:
if config.get('volumes'):
logging.info(f'local volume is being created')
@@ -141,6 +143,7 @@ def deploy_liminal_apps(path, clean):
liminal_home = environment.get_liminal_home()
os.makedirs(liminal_home, exist_ok=True)
os.makedirs(environment.get_dags_dir(), exist_ok=True)
+
deploy_liminal_core_internal(liminal_home, clean)
config_files = files_util.find_config_files(path)
for config_file in config_files:
@@ -150,6 +153,7 @@ def deploy_liminal_apps(path, clean):
pathlib.Path(target_yml_name).parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(config_file, target_yml_name)
+
def liminal_is_running():
stdout, stderr = docker_compose_command('ps', [])
return "liminal" in stdout
@@ -177,6 +181,7 @@ def logs(follow, tail):
stdout, stderr = docker_compose_command('logs', [f'--tail={tail}'])
logging.info(stdout)
+
@cli.command("start",
short_help="starts a local airflow in docker compose. should be run after deploy. " +
"Make sure docker is running on your machine")
@@ -193,13 +198,14 @@ def start(detached_mode):
else:
docker_compose_command('up', [])
+
@cli.command("delete", short_help="delete liminal resource registration")
@click.option('--path', default=os.getcwd(), help='Delete within this path.')
@click.option('--clean', is_flag=True, default=False,
help="Delete all resource: DAGs, Volumes")
def delete(path, clean):
configs = ConfigUtil(path).safe_load(is_render_variables=True,
- soft_merge=True)
+ soft_merge=True)
for config in configs:
if config.get('volumes'):
logging.info(f'local volume is being deleted')
@@ -211,6 +217,7 @@ def delete(path, clean):
shutil.rmtree(dir_path)
docker_compose_command('down', ['--remove-orphans', '--rmi', 'local'])
+
def get_docker_compose_paths():
# noinspection PyTypeChecker
with pkg_resources.path(scripts, 'docker-compose.yml') as p: