You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@liminal.apache.org by jb...@apache.org on 2020/07/20 06:25:21 UTC
[incubator-liminal] 41/43: Rename project to Liminal
This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 0c8f8dc994d41a5cab1b0c37142dfb1f6431faff
Author: aviemzur <av...@gmail.com>
AuthorDate: Tue Jun 23 15:20:36 2020 +0300
Rename project to Liminal
---
LICENSE | 4 +-
MANIFEST.in | 2 +-
README.md | 36 +++++++-------
images/{rainbow_001.png => liminal_001.png} | Bin
images/{rainbow_002.png => liminal_002.png} | Bin
rainbow-arch.md => liminal-arch.md | 18 +++----
.../rainbow/myserver => liminal}/__init__.py | 0
.../helloworld => liminal/build}/__init__.py | 0
.../rainbow => liminal/build/image}/__init__.py | 0
{rainbow => liminal}/build/image/python/Dockerfile | 0
.../sql => liminal/build/image/python}/__init__.py | 0
.../build/image/python/container-setup.sh | 4 +-
.../build/image/python/container-teardown.sh | 2 +-
{rainbow => liminal}/build/image/python/python.py | 2 +-
{rainbow => liminal}/build/image_builder.py | 4 +-
.../build/liminal_apps_builder.py | 18 +++----
{rainbow => liminal}/build/python.py | 2 +-
.../defaults => liminal/build/service}/__init__.py | 0
.../build/service/python_server/Dockerfile | 2 +-
.../build/service/python_server}/__init__.py | 0
.../service/python_server/liminal_python_server.py | 0
.../build/service/python_server/python_server.py | 6 +--
.../python_server/python_server_requirements.txt | 0
.../build/service/service_image_builder.py | 0
.../airflow/operators => liminal/core}/__init__.py | 0
{rainbow => liminal}/core/environment.py | 24 +++++-----
.../model => liminal/core/util}/__init__.py | 0
{rainbow => liminal}/core/util/class_util.py | 0
{rainbow => liminal}/core/util/files_util.py | 2 +-
.../airflow/dag => liminal/docker}/__init__.py | 0
.../airflow => liminal/monitoring}/__init__.py | 0
{rainbow => liminal}/runners/__init__.py | 0
.../runners/airflow}/__init__.py | 0
.../runners/airflow/config/__init__.py | 0
.../airflow/config/standalone_variable_backend.py | 8 ++--
.../runners/airflow/dag}/__init__.py | 0
.../runners/airflow/dag/liminal_dags.py | 24 +++++-----
.../runners/airflow/model}/__init__.py | 0
{rainbow => liminal}/runners/airflow/model/task.py | 0
.../runners/airflow/operators}/__init__.py | 0
.../runners/airflow/operators/cloudformation.py | 0
.../airflow/operators/job_status_operator.py | 0
.../kubernetes_pod_operator_with_input_output.py | 6 +--
.../runners/airflow/tasks}/__init__.py | 0
.../airflow/tasks/create_cloudformation_stack.py | 2 +-
.../runners/airflow/tasks/defaults}/__init__.py | 0
.../runners/airflow/tasks/defaults/default_task.py | 2 +-
.../runners/airflow/tasks/defaults/job_end.py | 4 +-
.../runners/airflow/tasks/defaults/job_start.py | 4 +-
.../airflow/tasks/delete_cloudformation_stack.py | 2 +-
.../runners/airflow/tasks/python.py | 6 +--
.../runners/airflow/tasks/spark.py | 2 +-
{rainbow => liminal}/runners/airflow/tasks/sql.py | 2 +-
.../build/image/python => liminal/sql}/__init__.py | 0
scripts/docker-compose.yml | 2 +-
scripts/{rainbow => liminal} | 52 +++++++++++----------
scripts/package.sh | 24 +++++-----
setup.py | 6 +--
.../python/test_python_server_image_builder.py | 6 +--
.../build/python/test_python_image_builder.py | 14 +++---
...ld_rainbows.py => test_liminal_apps_builder.py} | 9 ++--
.../{test_rainbow_dags.py => test_liminal_dags.py} | 8 ++--
.../runners/airflow/liminal}/__init__.py | 0
.../airflow/liminal/helloworld}/__init__.py | 0
.../{rainbow => liminal}/helloworld/hello_world.py | 4 +-
.../{rainbow/rainbow.yml => liminal/liminal.yml} | 0
.../runners/airflow/liminal/myserver}/__init__.py | 0
.../{rainbow => liminal}/myserver/my_server.py | 0
.../runners/airflow/{rainbow => liminal}/pip.conf | 0
.../airflow/{rainbow => liminal}/requirements.txt | 0
.../runners/airflow/tasks/defaults/test_job_end.py | 2 +-
.../airflow/tasks/defaults/test_job_start.py | 2 +-
tests/runners/airflow/tasks/test_python.py | 6 +--
tests/util/test_class_utils.py | 2 +-
74 files changed, 164 insertions(+), 161 deletions(-)
diff --git a/LICENSE b/LICENSE
index d9f1716..bdfa9f4 100644
--- a/LICENSE
+++ b/LICENSE
@@ -201,9 +201,9 @@ See the License for the specific language governing permissions and
limitations under the License.
============================================================================
- APACHE RAINBOW SUBCOMPONENTS:
+ APACHE LIMINAL SUBCOMPONENTS:
- The Apache Rainbow project contains subcomponents with separate copyright
+ The Apache Liminal project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.
diff --git a/MANIFEST.in b/MANIFEST.in
index 04cdb6d..6e3a162 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -18,4 +18,4 @@
include scripts/*
include requirements-airflow.txt
-recursive-include rainbow/build/ *
\ No newline at end of file
+recursive-include liminal/build/ *
\ No newline at end of file
diff --git a/README.md b/README.md
index 078343a..3623e76 100644
--- a/README.md
+++ b/README.md
@@ -17,14 +17,14 @@ specific language governing permissions and limitations
under the License.
-->
-# Rainbow
+# Liminal
-Rainbow is an end-to-end platform for data engineers & scientists, allowing them to build,
+Liminal is an end-to-end platform for data engineers & scientists, allowing them to build,
train and deploy machine learning models in a robust and agile way.
The platform provides the abstractions and declarative capabilities for
data extraction & feature engineering followed by model training and serving.
-Rainbow's goal is to operationalize the machine learning process, allowing data scientists to
+Liminal's goal is to operationalize the machine learning process, allowing data scientists to
quickly transition from a successful experiment to an automated pipeline of model training,
validation, deployment and inference in production, freeing them from engineering and
non-functional tasks, and allowing them to focus on machine learning code and artifacts.
@@ -101,20 +101,20 @@ services:
```bash
pip install liminal
```
-2. Optional: set RAINBOW_HOME to path of your choice (if not set, will default to ~/rainbow_home)
+2. Optional: set LIMINAL_HOME to path of your choice (if not set, will default to ~/liminal_home)
```bash
-echo 'export RAINBOW_HOME=</path/to/some/folder>' >> ~/.bash_profile && source ~/.bash_profile
+echo 'export LIMINAL_HOME=</path/to/some/folder>' >> ~/.bash_profile && source ~/.bash_profile
```
# Authoring pipelines
-This involves at minimum creating a single file called rainbow.yml as in the example above.
+This involves at minimum creating a single file called liminal.yml as in the example above.
If your pipeline requires custom python code to implement tasks, they should be organized
-[like this](https://github.com/Natural-Intelligence/rainbow/tree/master/tests/runners/airflow/rainbow)
+[like this](https://github.com/apache/incubator-liminal/tree/master/tests/runners/airflow/liminal)
If your pipeline introduces imports of external packages which are not already a part
-of the rainbow framework (i.e. you had to pip install them yourself), you need to also provide
+of the liminal framework (i.e. you had to pip install them yourself), you need to also provide
a requirements.txt in the root of your project.
# Testing the pipeline locally
@@ -122,7 +122,7 @@ a requirements.txt in the root of your project.
When your pipeline code is ready, you can test it by running it locally on your machine.
1. Ensure you have The Docker engine running locally, and enable a local Kubernetes cluster:
-
+
If you want to execute your pipeline on a remote kubernetes cluster, make sure the cluster is configured
using :
@@ -136,31 +136,31 @@ In the example pipeline above, you can see that tasks and services have an "imag
is created from a docker image where various code and libraries are installed.
You can take a look at what the build process looks like, e.g.
-[here](https://github.com/Natural-Intelligence/rainbow/tree/master/rainbow/build/image/python)
+[here](https://github.com/apache/incubator-liminal/tree/master/liminal/build/image/python)
In order for the images to be available for your pipeline, you'll need to build them locally:
```bash
-cd </path/to/your/rainbow/code>
-rainbow build
+cd </path/to/your/liminal/code>
+liminal build
```
You'll see that a number of outputs indicating various docker images built.
3. Deploy the pipeline:
```bash
-cd </path/to/your/rainbow/code>
-rainbow deploy
+cd </path/to/your/liminal/code>
+liminal deploy
```
4. Start the server
```bash
-rainbow start
+liminal start
```
5. Navigate to [http://localhost:8080/admin](http://localhost:8080/admin)
-6. You should see your 
+6. You should see your 
The pipeline is scheduled to run according to the ```json schedule: 0 * 1 * *``` field in the .yml file you provided.
7. To manually activate your pipeline:
@@ -168,7 +168,7 @@ Click your pipeline and then click "trigger DAG"
Click "Graph view"
You should see the steps in your pipeline getting executed in "real time" by clicking "Refresh" periodically.
-
+
### Running Tests (for contributors)
-When doing local development and running Rainbow unit-tests, make sure to set RAINBOW_STAND_ALONE_MODE=True
+When doing local development and running Liminal unit-tests, make sure to set LIMINAL_STAND_ALONE_MODE=True
diff --git a/images/rainbow_001.png b/images/liminal_001.png
similarity index 100%
rename from images/rainbow_001.png
rename to images/liminal_001.png
diff --git a/images/rainbow_002.png b/images/liminal_002.png
similarity index 100%
rename from images/rainbow_002.png
rename to images/liminal_002.png
diff --git a/rainbow-arch.md b/liminal-arch.md
similarity index 85%
rename from rainbow-arch.md
rename to liminal-arch.md
index 3e48712..835fb7c 100644
--- a/rainbow-arch.md
+++ b/liminal-arch.md
@@ -1,24 +1,24 @@
-# Rainbow
-Rainbow is an end-to-end platform for data engineers & scientists, allowing them to build, train and deploy machine learning models in a robust and agile way. The platform provides the abstractions and declarative capabilities for data extraction & feature engineering followed by model training and serving. Apache Rainbow's goal is to operationalise the machine learning process, allowing data scientists to quickly transition from a successful experiment to an automated pipeline of model [...]
+# Liminal
+Liminal is an end-to-end platform for data engineers & scientists, allowing them to build, train and deploy machine learning models in a robust and agile way. The platform provides the abstractions and declarative capabilities for data extraction & feature engineering followed by model training and serving. Apache Liminal's goal is to operationalise the machine learning process, allowing data scientists to quickly transition from a successful experiment to an automated pipeline of model [...]
## Motivation
The challenges involved in operationalizing machine learning models are one of the main reasons why many machine learning projects never make it to production.
The process involves automating and orchestrating multiple steps which run on heterogeneous infrastructure - different compute environments, data processing platforms, ML frameworks, notebooks, containers and monitoring tools.
There are no mature standards for this workflow, and most organizations do not have the experience to build it in-house. In the best case, dev-ds-devops teams form in order to accomplish this task together; in many cases, it's the data scientists who try to deal with this themselves without the knowledge or the inclination to become infrastructure experts.
As a result, many projects never make it through the cycle. Those who do suffer from a very long lead time from a successful experiment to an operational, refreshable, deployed and monitored model in production.
-The goal of Apache Rainbow is to simplify the creation and management of machine learning pipelines by data engineers & scientists. The platform provides declarative building blocks which define the workflow, orchestrate the underlying infrastructure, take care of non functional concerns, enabling focus in business logic / algorithm code.
+The goal of Apache Liminal is to simplify the creation and management of machine learning pipelines by data engineers & scientists. The platform provides declarative building blocks which define the workflow, orchestrate the underlying infrastructure, take care of non functional concerns, enabling focus in business logic / algorithm code.
Some Commercial E2E solutions have started to emerge in the last few years, however, they are limited to specific parts of the workflow, such as Databricks MLFlow. Other solutions are tied to specific environments (e.g. SageMaker on AWS).
## High Level Architecture
-The platform is aimed to provide data engineers & scientists with a solution for end to end flows from model training to real time inference in production. It’s architecture enables and promotes adoption of specific components in existing (non-Rainbow) frameworks, as well as seamless integration with other open source projects. Rainbow was created to enable scalability in ML efforts and after a thorough review of available solutions and frameworks, which did not meet our main KPIs:
+The platform is aimed to provide data engineers & scientists with a solution for end to end flows from model training to real time inference in production. It’s architecture enables and promotes adoption of specific components in existing (non-Liminal) frameworks, as well as seamless integration with other open source projects. Liminal was created to enable scalability in ML efforts and after a thorough review of available solutions and frameworks, which did not meet our main KPIs:
Provide an opinionated but customizable end-to-end workflow
Abstract away the complexity of underlying infrastructure
Support major open source tools and cloud-native infrastructure to carry out many of the steps
Allow teams to leverage their existing investments or bring in their tools of choice into the workflow
We have found that other tech companies in the Israeli Hi-Tech ecosystem also have an interest in such a platform, hence decided to share our work with the community.
-The following diagram depicts these main components and where Apache Rainbow comes in:
+The following diagram depicts these main components and where Apache Liminal comes in:
-
+
A classical data scientist workflow includes some base phases:
_Train, Deploy and Consume._
@@ -37,7 +37,7 @@ _Train, Deploy and Consume._
1. Inference - Batch or Real-time - use the model to evaluate data by your offline or online by your applications
1. Consume - The actual use of the models created by applications and ETLs, usually through APIs to the batch or real-time inference that usually rely on Model and Feature stores.
-Rainbow provides its users a declarative composition capabilities to materialize these steps in a robust way, while exploiting existing frameworks and tools. e.g. Data science frameworks such as scikit-learn, Tensor flow, Keras and such, for running core data science algorithms; as numerous core mechanisms as data stores, processing engines, parallelism, schedulers, code deployment as well as batch and real-time inference.
-Rainbow allows the creation and wiring of these kinds of functional and non functional tasks while making the underlying infrastructure used by these tasks very easy to use and even abstracted away entirely. While handling the non-functional aspects as monitoring (in a standard fashion) deployment, scheduling, resource management and execution.
+Liminal provides its users a declarative composition capabilities to materialize these steps in a robust way, while exploiting existing frameworks and tools. e.g. Data science frameworks such as scikit-learn, Tensor flow, Keras and such, for running core data science algorithms; as numerous core mechanisms as data stores, processing engines, parallelism, schedulers, code deployment as well as batch and real-time inference.
+Liminal allows the creation and wiring of these kinds of functional and non functional tasks while making the underlying infrastructure used by these tasks very easy to use and even abstracted away entirely. While handling the non-functional aspects as monitoring (in a standard fashion) deployment, scheduling, resource management and execution.
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/tests/runners/airflow/rainbow/myserver/__init__.py b/liminal/__init__.py
similarity index 100%
rename from tests/runners/airflow/rainbow/myserver/__init__.py
rename to liminal/__init__.py
diff --git a/tests/runners/airflow/rainbow/helloworld/__init__.py b/liminal/build/__init__.py
similarity index 100%
rename from tests/runners/airflow/rainbow/helloworld/__init__.py
rename to liminal/build/__init__.py
diff --git a/tests/runners/airflow/rainbow/__init__.py b/liminal/build/image/__init__.py
similarity index 100%
rename from tests/runners/airflow/rainbow/__init__.py
rename to liminal/build/image/__init__.py
diff --git a/rainbow/build/image/python/Dockerfile b/liminal/build/image/python/Dockerfile
similarity index 100%
rename from rainbow/build/image/python/Dockerfile
rename to liminal/build/image/python/Dockerfile
diff --git a/rainbow/sql/__init__.py b/liminal/build/image/python/__init__.py
similarity index 100%
rename from rainbow/sql/__init__.py
rename to liminal/build/image/python/__init__.py
diff --git a/rainbow/build/image/python/container-setup.sh b/liminal/build/image/python/container-setup.sh
similarity index 59%
rename from rainbow/build/image/python/container-setup.sh
rename to liminal/build/image/python/container-setup.sh
index c9e5cef..d8c98c5 100755
--- a/rainbow/build/image/python/container-setup.sh
+++ b/liminal/build/image/python/container-setup.sh
@@ -1,8 +1,8 @@
#!/bin/sh
-echo 'Writing rainbow input..'
+echo 'Writing liminal input..'
-echo """$RAINBOW_INPUT""" > /rainbow_input.json
+echo """$LIMINAL_INPUT""" > /liminal_input.json
AIRFLOW_RETURN_FILE=/airflow/xcom/return.json
diff --git a/rainbow/build/image/python/container-teardown.sh b/liminal/build/image/python/container-teardown.sh
similarity index 82%
rename from rainbow/build/image/python/container-teardown.sh
rename to liminal/build/image/python/container-teardown.sh
index 46c4426..2b2c23c 100755
--- a/rainbow/build/image/python/container-teardown.sh
+++ b/liminal/build/image/python/container-teardown.sh
@@ -1,6 +1,6 @@
#!/bin/sh
-echo 'Writing rainbow output..'
+echo 'Writing liminal output..'
USER_CONFIG_OUTPUT_FILE=$1
if [ "$USER_CONFIG_OUTPUT_FILE" != "" ]; then
diff --git a/rainbow/build/image/python/python.py b/liminal/build/image/python/python.py
similarity index 95%
rename from rainbow/build/image/python/python.py
rename to liminal/build/image/python/python.py
index 0ecec77..0d370d5 100644
--- a/rainbow/build/image/python/python.py
+++ b/liminal/build/image/python/python.py
@@ -18,7 +18,7 @@
import os
-from rainbow.build.python import BasePythonImageBuilder
+from liminal.build.python import BasePythonImageBuilder
class PythonImageBuilder(BasePythonImageBuilder):
diff --git a/rainbow/build/image_builder.py b/liminal/build/image_builder.py
similarity index 97%
rename from rainbow/build/image_builder.py
rename to liminal/build/image_builder.py
index a56a22e..3c300eb 100644
--- a/rainbow/build/image_builder.py
+++ b/liminal/build/image_builder.py
@@ -32,8 +32,8 @@ class ImageBuilder:
def __init__(self, config, base_path, relative_source_path, tag):
"""
:param config: task/service config
- :param base_path: directory containing rainbow yml
- :param relative_source_path: source path relative to rainbow yml
+ :param base_path: directory containing liminal yml
+ :param relative_source_path: source path relative to liminal yml
:param tag: image tag
"""
self.base_path = base_path
diff --git a/rainbow/build/build_rainbows.py b/liminal/build/liminal_apps_builder.py
similarity index 89%
rename from rainbow/build/build_rainbows.py
rename to liminal/build/liminal_apps_builder.py
index 66d27cb..efe2680 100644
--- a/rainbow/build/build_rainbows.py
+++ b/liminal/build/liminal_apps_builder.py
@@ -20,13 +20,13 @@ import os
import yaml
-from rainbow.build.image_builder import ImageBuilder, ServiceImageBuilderMixin
-from rainbow.core.util import files_util, class_util
+from liminal.build.image_builder import ImageBuilder, ServiceImageBuilderMixin
+from liminal.core.util import files_util, class_util
-def build_rainbows(path):
+def build_liminal_apps(path):
"""
- Build images for rainbows in path.
+ Build images for liminal apps in path.
"""
config_files = files_util.find_config_files(path)
@@ -36,9 +36,9 @@ def build_rainbows(path):
base_path = os.path.dirname(config_file)
with open(config_file) as stream:
- rainbow_config = yaml.safe_load(stream)
+ liminal_config = yaml.safe_load(stream)
- for pipeline in rainbow_config['pipelines']:
+ for pipeline in liminal_config['pipelines']:
for task in pipeline['tasks']:
task_name = task['task']
@@ -52,7 +52,7 @@ def build_rainbows(path):
else:
print(f'No source configured for task {task_name}, skipping build..')
- for service in rainbow_config['services']:
+ for service in liminal_config['services']:
service_type = service['type']
builder_class = __get_service_build_class(service_type)
if builder_class:
@@ -84,7 +84,7 @@ def __get_service_build_class(service_type):
print(f'Loading image builder implementations..')
# TODO: add configuration for user image builders package
-image_builders_package = 'rainbow.build.image'
+image_builders_package = 'liminal.build.image'
# user_image_builders_package = 'TODO: user_image_builders_package'
task_build_classes = class_util.find_subclasses_in_packages(
@@ -104,7 +104,7 @@ print(f'Finished loading image builder implementations: {task_build_classes}')
print(f'Loading service image builder implementations..')
# TODO: add configuration for user service image builders package
-service_builders_package = 'rainbow.build.service'
+service_builders_package = 'liminal.build.service'
# user_service_builders_package = 'TODO: user_service_builders_package'
service_build_classes = class_util.find_subclasses_in_packages(
diff --git a/rainbow/build/python.py b/liminal/build/python.py
similarity index 97%
rename from rainbow/build/python.py
rename to liminal/build/python.py
index 0961d2b..07c6155 100644
--- a/rainbow/build/python.py
+++ b/liminal/build/python.py
@@ -18,7 +18,7 @@
import os
-from rainbow.build.image_builder import ImageBuilder
+from liminal.build.image_builder import ImageBuilder
class BasePythonImageBuilder(ImageBuilder):
diff --git a/rainbow/runners/airflow/tasks/defaults/__init__.py b/liminal/build/service/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/tasks/defaults/__init__.py
rename to liminal/build/service/__init__.py
diff --git a/rainbow/build/service/python_server/Dockerfile b/liminal/build/service/python_server/Dockerfile
similarity index 97%
rename from rainbow/build/service/python_server/Dockerfile
rename to liminal/build/service/python_server/Dockerfile
index e329738..96c2e4a 100644
--- a/rainbow/build/service/python_server/Dockerfile
+++ b/liminal/build/service/python_server/Dockerfile
@@ -41,4 +41,4 @@ RUN {{mount}} pip install -r requirements.txt
RUN echo "Copying source code.."
COPY . /app/
-CMD python -u rainbow_python_server.py
+CMD python -u liminal_python_server.py
diff --git a/rainbow/runners/airflow/tasks/__init__.py b/liminal/build/service/python_server/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/tasks/__init__.py
rename to liminal/build/service/python_server/__init__.py
diff --git a/rainbow/build/service/python_server/rainbow_python_server.py b/liminal/build/service/python_server/liminal_python_server.py
similarity index 100%
rename from rainbow/build/service/python_server/rainbow_python_server.py
rename to liminal/build/service/python_server/liminal_python_server.py
diff --git a/rainbow/build/service/python_server/python_server.py b/liminal/build/service/python_server/python_server.py
similarity index 89%
rename from rainbow/build/service/python_server/python_server.py
rename to liminal/build/service/python_server/python_server.py
index 0b2537d..f0d5b99 100644
--- a/rainbow/build/service/python_server/python_server.py
+++ b/liminal/build/service/python_server/python_server.py
@@ -20,8 +20,8 @@ import os
import yaml
-from rainbow.build.image_builder import ServiceImageBuilderMixin
-from rainbow.build.python import BasePythonImageBuilder
+from liminal.build.image_builder import ServiceImageBuilderMixin
+from liminal.build.python import BasePythonImageBuilder
class PythonServerImageBuilder(BasePythonImageBuilder, ServiceImageBuilderMixin):
@@ -36,7 +36,7 @@ class PythonServerImageBuilder(BasePythonImageBuilder, ServiceImageBuilderMixin)
@staticmethod
def _additional_files_from_paths():
return [
- os.path.join(os.path.dirname(__file__), 'rainbow_python_server.py'),
+ os.path.join(os.path.dirname(__file__), 'liminal_python_server.py'),
os.path.join(os.path.dirname(__file__), 'python_server_requirements.txt')
]
diff --git a/rainbow/build/service/python_server/python_server_requirements.txt b/liminal/build/service/python_server/python_server_requirements.txt
similarity index 100%
rename from rainbow/build/service/python_server/python_server_requirements.txt
rename to liminal/build/service/python_server/python_server_requirements.txt
diff --git a/rainbow/build/service/service_image_builder.py b/liminal/build/service/service_image_builder.py
similarity index 100%
rename from rainbow/build/service/service_image_builder.py
rename to liminal/build/service/service_image_builder.py
diff --git a/rainbow/runners/airflow/operators/__init__.py b/liminal/core/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/operators/__init__.py
rename to liminal/core/__init__.py
diff --git a/rainbow/core/environment.py b/liminal/core/environment.py
similarity index 61%
rename from rainbow/core/environment.py
rename to liminal/core/environment.py
index 27f4d41..f335ce9 100644
--- a/rainbow/core/environment.py
+++ b/liminal/core/environment.py
@@ -18,21 +18,21 @@
import os
-DEFAULT_DAGS_ZIP_NAME = 'rainbows.zip'
-DEFAULT_RAINBOW_HOME = os.path.expanduser('~/rainbow_home')
-DEFAULT_RAINBOWS_SUBDIR = "rainbows"
-RAINBOW_HOME_PARAM_NAME = "RAINBOW_HOME"
+DEFAULT_DAGS_ZIP_NAME = 'liminal.zip'
+DEFAULT_LIMINAL_HOME = os.path.expanduser('~/liminal_home')
+DEFAULT_PIPELINES_SUBDIR = "pipelines"
+LIMINAL_HOME_PARAM_NAME = "LIMINAL_HOME"
-def get_rainbow_home():
- if not os.environ.get(RAINBOW_HOME_PARAM_NAME):
- print("no environment parameter called RAINBOW_HOME detected")
- print(f"registering {DEFAULT_RAINBOW_HOME} as the RAINBOW_HOME directory")
- os.environ[RAINBOW_HOME_PARAM_NAME] = DEFAULT_RAINBOW_HOME
- return os.environ.get(RAINBOW_HOME_PARAM_NAME, DEFAULT_RAINBOW_HOME)
+def get_liminal_home():
+ if not os.environ.get(LIMINAL_HOME_PARAM_NAME):
+ print("no environment parameter called LIMINAL_HOME detected")
+ print(f"registering {DEFAULT_LIMINAL_HOME} as the LIMINAL_HOME directory")
+ os.environ[LIMINAL_HOME_PARAM_NAME] = DEFAULT_LIMINAL_HOME
+ return os.environ.get(LIMINAL_HOME_PARAM_NAME, DEFAULT_LIMINAL_HOME)
def get_dags_dir():
# if we are inside airflow, we will take it from the configured dags folder
- base_dir = os.environ.get("AIRFLOW__CORE__DAGS_FOLDER", get_rainbow_home())
- return os.path.join(base_dir, DEFAULT_RAINBOWS_SUBDIR)
+ base_dir = os.environ.get("AIRFLOW__CORE__DAGS_FOLDER", get_liminal_home())
+ return os.path.join(base_dir, DEFAULT_PIPELINES_SUBDIR)
diff --git a/rainbow/runners/airflow/model/__init__.py b/liminal/core/util/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/model/__init__.py
rename to liminal/core/util/__init__.py
diff --git a/rainbow/core/util/class_util.py b/liminal/core/util/class_util.py
similarity index 100%
rename from rainbow/core/util/class_util.py
rename to liminal/core/util/class_util.py
diff --git a/rainbow/core/util/files_util.py b/liminal/core/util/files_util.py
similarity index 93%
rename from rainbow/core/util/files_util.py
rename to liminal/core/util/files_util.py
index 4a03020..e611005 100644
--- a/rainbow/core/util/files_util.py
+++ b/liminal/core/util/files_util.py
@@ -24,7 +24,7 @@ def find_config_files(path):
print(path)
for r, d, f in os.walk(path):
for file in f:
- if os.path.basename(file) in ['rainbow.yml', 'rainbow.yaml']:
+ if os.path.basename(file) in ['liminal.yml', 'liminal.yaml']:
print(os.path.join(r, file))
files.append(os.path.join(r, file))
return files
diff --git a/rainbow/runners/airflow/dag/__init__.py b/liminal/docker/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/dag/__init__.py
rename to liminal/docker/__init__.py
diff --git a/rainbow/runners/airflow/__init__.py b/liminal/monitoring/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/__init__.py
rename to liminal/monitoring/__init__.py
diff --git a/rainbow/runners/__init__.py b/liminal/runners/__init__.py
similarity index 100%
rename from rainbow/runners/__init__.py
rename to liminal/runners/__init__.py
diff --git a/rainbow/monitoring/__init__.py b/liminal/runners/airflow/__init__.py
similarity index 100%
rename from rainbow/monitoring/__init__.py
rename to liminal/runners/airflow/__init__.py
diff --git a/rainbow/runners/airflow/config/__init__.py b/liminal/runners/airflow/config/__init__.py
similarity index 100%
rename from rainbow/runners/airflow/config/__init__.py
rename to liminal/runners/airflow/config/__init__.py
diff --git a/rainbow/runners/airflow/config/standalone_variable_backend.py b/liminal/runners/airflow/config/standalone_variable_backend.py
similarity index 85%
rename from rainbow/runners/airflow/config/standalone_variable_backend.py
rename to liminal/runners/airflow/config/standalone_variable_backend.py
index d7df06c..b49c7fe 100644
--- a/rainbow/runners/airflow/config/standalone_variable_backend.py
+++ b/liminal/runners/airflow/config/standalone_variable_backend.py
@@ -20,16 +20,16 @@ from os import environ
from airflow.models import Variable
-RAINBOW_STAND_ALONE_MODE_KEY = "RAINBOW_STAND_ALONE_MODE"
+LIMINAL_STAND_ALONE_MODE_KEY = "LIMINAL_STAND_ALONE_MODE"
def get_variable(key, default_val):
- if rainbow_local_mode():
+ if liminal_local_mode():
return os.environ.get(key, default_val)
else:
return Variable.get(key, default_var=default_val)
-def rainbow_local_mode():
- stand_alone = environ.get(RAINBOW_STAND_ALONE_MODE_KEY, "False")
+def liminal_local_mode():
+ stand_alone = environ.get(LIMINAL_STAND_ALONE_MODE_KEY, "False")
return stand_alone.strip().lower() == "true"
diff --git a/rainbow/docker/__init__.py b/liminal/runners/airflow/dag/__init__.py
similarity index 100%
rename from rainbow/docker/__init__.py
rename to liminal/runners/airflow/dag/__init__.py
diff --git a/rainbow/runners/airflow/dag/rainbow_dags.py b/liminal/runners/airflow/dag/liminal_dags.py
similarity index 85%
rename from rainbow/runners/airflow/dag/rainbow_dags.py
rename to liminal/runners/airflow/dag/liminal_dags.py
index 9deef8e..9798ed4 100644
--- a/rainbow/runners/airflow/dag/rainbow_dags.py
+++ b/liminal/runners/airflow/dag/liminal_dags.py
@@ -23,19 +23,19 @@ import yaml
from airflow import DAG
from airflow.models import Variable
-from rainbow.core import environment
-from rainbow.core.util import class_util
-from rainbow.core.util import files_util
-from rainbow.runners.airflow.model.task import Task
-from rainbow.runners.airflow.tasks.defaults.job_end import JobEndTask
-from rainbow.runners.airflow.tasks.defaults.job_start import JobStartTask
+from liminal.core import environment
+from liminal.core.util import class_util
+from liminal.core.util import files_util
+from liminal.runners.airflow.model.task import Task
+from liminal.runners.airflow.tasks.defaults.job_end import JobEndTask
+from liminal.runners.airflow.tasks.defaults.job_start import JobStartTask
__DEPENDS_ON_PAST = 'depends_on_past'
def register_dags(configs_path):
"""
- Registers pipelines in rainbow yml files found in given path (recursively) as airflow DAGs.
+ Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs.
"""
print(f'Registering DAG from path: {configs_path}')
config_files = files_util.find_config_files(configs_path)
@@ -96,24 +96,24 @@ def register_dags(configs_path):
print(f'Loading task implementations..')
# TODO: add configuration for user tasks package
-impl_packages = 'rainbow.runners.airflow.tasks'
+impl_packages = 'liminal.runners.airflow.tasks'
user_task_package = 'TODO: user_tasks_package'
task_classes = class_util.find_subclasses_in_packages([impl_packages], Task)
-def tasks_by_rainbow_name(task_classes):
+def tasks_by_liminal_name(task_classes):
return {full_name.replace(impl_packages, '').replace(clzz.__name__, '')[1:-1]: clzz
for (full_name, clzz) in task_classes.items()}
-tasks_by_rainbow_name = tasks_by_rainbow_name(task_classes)
+tasks_by_liminal_name = tasks_by_liminal_name(task_classes)
-print(f'Finished loading task implementations: {tasks_by_rainbow_name}')
+print(f'Finished loading task implementations: {tasks_by_liminal_name}')
def get_task_class(task_type):
- return tasks_by_rainbow_name[task_type]
+ return tasks_by_liminal_name[task_type]
register_dags(environment.get_dags_dir())
diff --git a/rainbow/core/util/__init__.py b/liminal/runners/airflow/model/__init__.py
similarity index 100%
rename from rainbow/core/util/__init__.py
rename to liminal/runners/airflow/model/__init__.py
diff --git a/rainbow/runners/airflow/model/task.py b/liminal/runners/airflow/model/task.py
similarity index 100%
rename from rainbow/runners/airflow/model/task.py
rename to liminal/runners/airflow/model/task.py
diff --git a/rainbow/core/__init__.py b/liminal/runners/airflow/operators/__init__.py
similarity index 100%
rename from rainbow/core/__init__.py
rename to liminal/runners/airflow/operators/__init__.py
diff --git a/rainbow/runners/airflow/operators/cloudformation.py b/liminal/runners/airflow/operators/cloudformation.py
similarity index 100%
rename from rainbow/runners/airflow/operators/cloudformation.py
rename to liminal/runners/airflow/operators/cloudformation.py
diff --git a/rainbow/runners/airflow/operators/job_status_operator.py b/liminal/runners/airflow/operators/job_status_operator.py
similarity index 100%
rename from rainbow/runners/airflow/operators/job_status_operator.py
rename to liminal/runners/airflow/operators/job_status_operator.py
diff --git a/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py b/liminal/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
similarity index 95%
rename from rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
rename to liminal/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
index c44e80b..5833550 100644
--- a/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
+++ b/liminal/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py
@@ -93,7 +93,7 @@ class KubernetesPodOperatorWithInputAndOutput(KubernetesPodOperator):
TODO: pydoc
"""
- _RAINBOW_INPUT_ENV_VAR = 'RAINBOW_INPUT'
+ _LIMINAL_INPUT_ENV_VAR = 'LIMINAL_INPUT'
def __init__(self,
task_split,
@@ -138,9 +138,9 @@ class KubernetesPodOperatorWithInputAndOutput(KubernetesPodOperator):
if task_input:
self.log.info(f'task input = {task_input}')
- self.env_vars.update({self._RAINBOW_INPUT_ENV_VAR: json.dumps(task_input)})
+ self.env_vars.update({self._LIMINAL_INPUT_ENV_VAR: json.dumps(task_input)})
else:
- self.env_vars.update({self._RAINBOW_INPUT_ENV_VAR: '{}'})
+ self.env_vars.update({self._LIMINAL_INPUT_ENV_VAR: '{}'})
self.log.info(f'Empty input for task {self.task_split}.')
diff --git a/rainbow/build/service/python_server/__init__.py b/liminal/runners/airflow/tasks/__init__.py
similarity index 100%
rename from rainbow/build/service/python_server/__init__.py
rename to liminal/runners/airflow/tasks/__init__.py
diff --git a/rainbow/runners/airflow/tasks/create_cloudformation_stack.py b/liminal/runners/airflow/tasks/create_cloudformation_stack.py
similarity index 95%
rename from rainbow/runners/airflow/tasks/create_cloudformation_stack.py
rename to liminal/runners/airflow/tasks/create_cloudformation_stack.py
index 8f069f3..01bbba1 100644
--- a/rainbow/runners/airflow/tasks/create_cloudformation_stack.py
+++ b/liminal/runners/airflow/tasks/create_cloudformation_stack.py
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-from rainbow.runners.airflow.model import task
+from liminal.runners.airflow.model import task
class CreateCloudFormationStackTask(task.Task):
diff --git a/rainbow/build/service/__init__.py b/liminal/runners/airflow/tasks/defaults/__init__.py
similarity index 100%
rename from rainbow/build/service/__init__.py
rename to liminal/runners/airflow/tasks/defaults/__init__.py
diff --git a/rainbow/runners/airflow/tasks/defaults/default_task.py b/liminal/runners/airflow/tasks/defaults/default_task.py
similarity index 96%
rename from rainbow/runners/airflow/tasks/defaults/default_task.py
rename to liminal/runners/airflow/tasks/defaults/default_task.py
index 0e901fc..0a57284 100644
--- a/rainbow/runners/airflow/tasks/defaults/default_task.py
+++ b/liminal/runners/airflow/tasks/defaults/default_task.py
@@ -20,7 +20,7 @@ Default base task.
"""
from abc import abstractmethod
-from rainbow.runners.airflow.model.task import Task
+from liminal.runners.airflow.model.task import Task
class DefaultTask(Task):
diff --git a/rainbow/runners/airflow/tasks/defaults/job_end.py b/liminal/runners/airflow/tasks/defaults/job_end.py
similarity index 92%
rename from rainbow/runners/airflow/tasks/defaults/job_end.py
rename to liminal/runners/airflow/tasks/defaults/job_end.py
index e177ccc..f90b482 100644
--- a/rainbow/runners/airflow/tasks/defaults/job_end.py
+++ b/liminal/runners/airflow/tasks/defaults/job_end.py
@@ -16,8 +16,8 @@
# specific language governing permissions and limitations
# under the License.
-from rainbow.runners.airflow.operators.job_status_operator import JobEndOperator
-from rainbow.runners.airflow.tasks.defaults.default_task import DefaultTask
+from liminal.runners.airflow.operators.job_status_operator import JobEndOperator
+from liminal.runners.airflow.tasks.defaults.default_task import DefaultTask
class JobEndTask(DefaultTask):
diff --git a/rainbow/runners/airflow/tasks/defaults/job_start.py b/liminal/runners/airflow/tasks/defaults/job_start.py
similarity index 92%
rename from rainbow/runners/airflow/tasks/defaults/job_start.py
rename to liminal/runners/airflow/tasks/defaults/job_start.py
index e196919..3c1d831 100644
--- a/rainbow/runners/airflow/tasks/defaults/job_start.py
+++ b/liminal/runners/airflow/tasks/defaults/job_start.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from rainbow.runners.airflow.operators.job_status_operator import JobStartOperator
-from rainbow.runners.airflow.tasks.defaults.default_task import DefaultTask
+from liminal.runners.airflow.operators.job_status_operator import JobStartOperator
+from liminal.runners.airflow.tasks.defaults.default_task import DefaultTask
class JobStartTask(DefaultTask):
diff --git a/rainbow/runners/airflow/tasks/delete_cloudformation_stack.py b/liminal/runners/airflow/tasks/delete_cloudformation_stack.py
similarity index 95%
rename from rainbow/runners/airflow/tasks/delete_cloudformation_stack.py
rename to liminal/runners/airflow/tasks/delete_cloudformation_stack.py
index ab99101..b2347e7 100644
--- a/rainbow/runners/airflow/tasks/delete_cloudformation_stack.py
+++ b/liminal/runners/airflow/tasks/delete_cloudformation_stack.py
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-from rainbow.runners.airflow.model import task
+from liminal.runners.airflow.model import task
class DeleteCloudFormationStackTask(task.Task):
diff --git a/rainbow/runners/airflow/tasks/python.py b/liminal/runners/airflow/tasks/python.py
similarity index 97%
rename from rainbow/runners/airflow/tasks/python.py
rename to liminal/runners/airflow/tasks/python.py
index d5d4e00..9d8635f 100644
--- a/rainbow/runners/airflow/tasks/python.py
+++ b/liminal/runners/airflow/tasks/python.py
@@ -20,9 +20,9 @@ import json
from airflow.models import Variable
from airflow.operators.dummy_operator import DummyOperator
-from rainbow.runners.airflow.config.standalone_variable_backend import get_variable
-from rainbow.runners.airflow.model import task
-from rainbow.runners.airflow.operators.kubernetes_pod_operator_with_input_output import \
+from liminal.runners.airflow.config.standalone_variable_backend import get_variable
+from liminal.runners.airflow.model import task
+from liminal.runners.airflow.operators.kubernetes_pod_operator_with_input_output import \
KubernetesPodOperatorWithInputAndOutput, \
PrepareInputOperator
diff --git a/rainbow/runners/airflow/tasks/spark.py b/liminal/runners/airflow/tasks/spark.py
similarity index 95%
rename from rainbow/runners/airflow/tasks/spark.py
rename to liminal/runners/airflow/tasks/spark.py
index 68cfac0..a93722e 100644
--- a/rainbow/runners/airflow/tasks/spark.py
+++ b/liminal/runners/airflow/tasks/spark.py
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-from rainbow.runners.airflow.model import task
+from liminal.runners.airflow.model import task
class SparkTask(task.Task):
"""
diff --git a/rainbow/runners/airflow/tasks/sql.py b/liminal/runners/airflow/tasks/sql.py
similarity index 95%
rename from rainbow/runners/airflow/tasks/sql.py
rename to liminal/runners/airflow/tasks/sql.py
index 7ae3b9f..f15e356 100644
--- a/rainbow/runners/airflow/tasks/sql.py
+++ b/liminal/runners/airflow/tasks/sql.py
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-from rainbow.runners.airflow.model import task
+from liminal.runners.airflow.model import task
class SparkTask(task.Task):
diff --git a/rainbow/build/image/python/__init__.py b/liminal/sql/__init__.py
similarity index 100%
rename from rainbow/build/image/python/__init__.py
rename to liminal/sql/__init__.py
diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index d0304e5..8cab3ea 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -29,7 +29,7 @@
max-size: 10m
max-file: "3"
volumes:
- - ${RAINBOW_HOME}:/usr/local/airflow/dags
+ - ${LIMINAL_HOME}:/usr/local/airflow/dags
- ${HOME}/.kube:/usr/local/airflow/.kube
ports:
- "8080:8080"
diff --git a/scripts/rainbow b/scripts/liminal
similarity index 60%
rename from scripts/rainbow
rename to scripts/liminal
index 1d5f65e..5ea21ad 100755
--- a/scripts/rainbow
+++ b/scripts/liminal
@@ -19,14 +19,14 @@
# under the License.
import os
import shutil
-import site
+import subprocess
import sys
import click
-from rainbow.build import build_rainbows
-import subprocess
-from rainbow.core import environment
-from rainbow.core.util import files_util
+
+from liminal.build import liminal_apps_builder
+from liminal.core import environment
+from liminal.core.util import files_util
@click.group()
@@ -46,41 +46,43 @@ def docker_is_running():
@cli.command("build", short_help="builds dockers from your business logic")
@click.option('--path', default=os.getcwd(), help='Build within this path.')
def build(path):
- click.echo(f'Building rainbows in {path}')
+ click.echo(f'Building liminal apps in {path}')
if docker_is_running():
- build_rainbows.build_rainbows(path)
+ liminal_apps_builder.build_liminal_apps(path)
-def deploy_rainbow_core_internal():
- click.echo("WARN: refreshing rainbow core package")
- rainbow_home = environment.get_rainbow_home()
- subprocess.call([f'package.sh {rainbow_home}'], shell=True)
+def deploy_liminal_core_internal():
+ click.echo("WARN: refreshing liminal core package")
+ liminal_home = environment.get_liminal_home()
+ subprocess.call([f'package.sh {liminal_home}'], shell=True)
-@cli.command("deploy", short_help="deploys your rainbow.yaml files to $RAINBOW_HOME folder")
-@click.option('--path', default=os.getcwd(), help="folder containing rainbow.yaml files")
-def deploy_rainbows(path):
- click.echo("deploying rainbow yaml files")
- rainbow_home = environment.get_rainbow_home()
- os.makedirs(rainbow_home, exist_ok=True)
+@cli.command("deploy", short_help="deploys your liminal.yaml files to $LIMINAL_HOME folder")
+@click.option('--path', default=os.getcwd(), help="folder containing liminal.yaml files")
+def deploy_liminal_apps(path):
+ click.echo("deploying liminal yaml files")
+ liminal_home = environment.get_liminal_home()
+ os.makedirs(liminal_home, exist_ok=True)
os.makedirs(environment.get_dags_dir(), exist_ok=True)
- deploy_rainbow_core_internal()
+ deploy_liminal_core_internal()
config_files = files_util.find_config_files(path)
for config_file in config_files:
- click.echo(f"deploying rainbow file: {config_file}")
+ click.echo(f"deploying liminal file: {config_file}")
yml_name = os.path.basename(config_file)
target_yml_name = os.path.join(environment.get_dags_dir(), yml_name)
shutil.copyfile(config_file, target_yml_name)
-@cli.command("start", short_help="starts a local airflow in docker compose. should be run after deploy. " +
- "Make sure docker is running on your machine")
+@cli.command("start",
+ short_help="starts a local airflow in docker compose. should be run after deploy. " +
+ "Make sure docker is running on your machine")
def start():
if docker_is_running():
- # initialize rainbow home by default
- environment.get_rainbow_home()
- result = subprocess.call([f'docker-compose -f "{environment.get_rainbow_home()}/docker-compose.yml" up'],
- env=os.environ, shell=True)
+ # initialize liminal home by default
+ environment.get_liminal_home()
+ result = subprocess.call(
+ [f'docker-compose -f "{environment.get_liminal_home()}/docker-compose.yml" up'],
+ env=os.environ, shell=True)
if __name__ == '__main__':
diff --git a/scripts/package.sh b/scripts/package.sh
index 7824fd5..de88f41 100755
--- a/scripts/package.sh
+++ b/scripts/package.sh
@@ -21,12 +21,12 @@ echo $1
target_path="$1"
echo "running from " $(PWD)
-echo "target path for rainbow zip file is " $target_path
+echo "target path for liminal zip file is " $target_path
-echo "cleaning up the temp dirs $TMPDIR/rainbow_build"
-rm -rf $TMPDIR/rainbow_build-*/
+echo "cleaning up the temp dirs $TMPDIR/liminal_build"
+rm -rf $TMPDIR/liminal_build-*/
-tmp_dir=$(mktemp -d -t rainbow_build-)
+tmp_dir=$(mktemp -d -t liminal_build-)
echo "creating temp directory $tmp_dir"
docker_build_dir=$tmp_dir/docker_build
@@ -43,12 +43,12 @@ rsync -a --exclude 'venv' $(PWD)/ $docker_build_dir/zip_content/
# this is done inside a docker to 1) avoid requiring the user to install stuff, and 2) to create a platform-compatible
# package (install the native libraries in a flavour suitable for the docker in which airflow runs, and not user machine)
-docker run --rm --name rainbow_build -v /private/"$docker_build_dir":/home/rainbow/tmp --entrypoint="" -u 0 \
- puckel/docker-airflow:1.10.9 /bin/bash -c "cd /home/rainbow/tmp/zip_content &&
- pip install --no-deps --target=\"/home/rainbow/tmp/zip_content\" liminal==0.0.2dev5 &&
- rsync -avzh --ignore-errors /home/rainbow/tmp/zip_content/liminal-resources/* /home/rainbow/tmp/zip_content/
- pip install --target=\"/home/rainbow/tmp/zip_content\" -r /home/rainbow/tmp/zip_content/requirements-airflow.txt &&
- pip install --target=\"/home/rainbow/tmp/zip_content\" -r /home/rainbow/tmp/zip_content/requirements.txt"
+docker run --rm --name liminal_build -v /private/"$docker_build_dir":/home/liminal/tmp --entrypoint="" -u 0 \
+ puckel/docker-airflow:1.10.9 /bin/bash -c "cd /home/liminal/tmp/zip_content &&
+ pip install --no-deps --target=\"/home/liminal/tmp/zip_content\" liminal==0.0.2dev5 &&
+ rsync -avzh --ignore-errors /home/liminal/tmp/zip_content/liminal-resources/* /home/liminal/tmp/zip_content/
+ pip install --target=\"/home/liminal/tmp/zip_content\" -r /home/liminal/tmp/zip_content/requirements-airflow.txt &&
+ pip install --target=\"/home/liminal/tmp/zip_content\" -r /home/liminal/tmp/zip_content/requirements.txt"
# zip the content per https://airflow.apache.org/docs/stable/concepts.html#packaged-dags
@@ -56,5 +56,5 @@ cd $docker_build_dir/zip_content
mv docker-compose.yml $target_path
rm __init__.py
-zip -r ../dags/rainbows.zip .
-cp ../dags/rainbows.zip $target_path
+zip -r ../dags/liminal.zip .
+cp ../dags/liminal.zip $target_path
diff --git a/setup.py b/setup.py
index 2a0fdbb..0cf0e64 100644
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,7 @@ setuptools.setup(
description="A package for authoring and deploying machine learning workflows",
long_description=long_description,
long_description_content_type="text/markdown",
- url="https://github.com/Natural-Intelligence/rainbow",
+ url="https://github.com/apache/incubator-liminal",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
@@ -45,9 +45,9 @@ setuptools.setup(
license='Apache License, Version 2.0',
python_requires='>=3.6',
install_requires=requirements,
- scripts=['scripts/rainbow', 'scripts/package.sh'],
+ scripts=['scripts/liminal', 'scripts/package.sh'],
include_package_data=True,
data_files=[('liminal-resources', ['scripts/docker-compose.yml',
'requirements-airflow.txt',
- 'rainbow/runners/airflow/dag/rainbow_dags.py'])]
+ 'liminal/runners/airflow/dag/liminal_dags.py'])]
)
diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
index ecdaced..145d931 100644
--- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
+++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
@@ -25,7 +25,7 @@ from unittest import TestCase
import docker
-from rainbow.build.service.python_server.python_server import PythonServerImageBuilder
+from liminal.build.service.python_server.python_server import PythonServerImageBuilder
class TestPythonServer(TestCase):
@@ -54,7 +54,7 @@ class TestPythonServer(TestCase):
'Incorrect pip command')
def __test_build_python_server(self, use_pip_conf=False):
- base_path = os.path.join(os.path.dirname(__file__), '../../../rainbow')
+ base_path = os.path.join(os.path.dirname(__file__), '../../../liminal')
config = self.__create_conf('my_task')
@@ -112,7 +112,7 @@ class TestPythonServer(TestCase):
return {
'task': task_id,
'cmd': 'foo bar',
- 'image': 'rainbow_server_image',
+ 'image': 'liminal_server_image',
'source': 'baz',
'input_type': 'my_input_type',
'input_path': 'my_input',
diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py
index 81b5cc3..ab25dab 100644
--- a/tests/runners/airflow/build/python/test_python_image_builder.py
+++ b/tests/runners/airflow/build/python/test_python_image_builder.py
@@ -22,11 +22,11 @@ from unittest import TestCase
import docker
-from rainbow.build.image.python.python import PythonImageBuilder
+from liminal.build.image.python.python import PythonImageBuilder
class TestPythonImageBuilder(TestCase):
- __IMAGE_NAME = 'rainbow_image'
+ __IMAGE_NAME = 'liminal_image'
__OUTPUT_PATH = '/mnt/vol1/my_output.json'
def setUp(self) -> None:
@@ -59,7 +59,7 @@ class TestPythonImageBuilder(TestCase):
def __test_build(self, use_pip_conf=False):
config = self.__create_conf('my_task')
- base_path = os.path.join(os.path.dirname(__file__), '../../rainbow')
+ base_path = os.path.join(os.path.dirname(__file__), '../../liminal')
if use_pip_conf:
config['pip_conf'] = os.path.join(base_path, 'pip.conf')
@@ -77,7 +77,7 @@ class TestPythonImageBuilder(TestCase):
docker_client = docker.from_env()
docker_client.images.get(self.__IMAGE_NAME)
- cmd = 'export RAINBOW_INPUT="{\\"x\\": 1}" && ' + \
+ cmd = 'export LIMINAL_INPUT="{\\"x\\": 1}" && ' + \
'sh container-setup.sh && ' + \
'python hello_world.py && ' + \
f'sh container-teardown.sh {self.__OUTPUT_PATH}'
@@ -100,10 +100,10 @@ class TestPythonImageBuilder(TestCase):
print(container_log)
self.assertEqual(
- "b\"Writing rainbow input..\\n" +
+ "b\"Writing liminal input..\\n" +
"Hello world!\\n\\n" +
- "rainbow_input.json contents = {'x': 1}\\n" +
- "Writing rainbow output..\\n\"",
+ "liminal_input.json contents = {'x': 1}\\n" +
+ "Writing liminal output..\\n\"",
str(container_log))
with open(os.path.join(self.temp_airflow_dir, 'return.json')) as file:
diff --git a/tests/runners/airflow/build/test_build_rainbows.py b/tests/runners/airflow/build/test_liminal_apps_builder.py
similarity index 86%
rename from tests/runners/airflow/build/test_build_rainbows.py
rename to tests/runners/airflow/build/test_liminal_apps_builder.py
index 7e01245..6888eb6 100644
--- a/tests/runners/airflow/build/test_build_rainbows.py
+++ b/tests/runners/airflow/build/test_liminal_apps_builder.py
@@ -21,10 +21,10 @@ from unittest import TestCase
import docker
-from rainbow.build import build_rainbows
+from liminal.build import liminal_apps_builder
-class TestBuildRainbows(TestCase):
+class TestLiminalAppsBuilder(TestCase):
__image_names = [
'my_static_input_task_image',
'my_task_output_input_task_image',
@@ -44,8 +44,9 @@ class TestBuildRainbows(TestCase):
if len(self.docker_client.images.list(image_name)) > 0:
self.docker_client.images.remove(image=image_name, force=True)
- def test_build_rainbow(self):
- build_rainbows.build_rainbows(os.path.join(os.path.dirname(__file__), '../rainbow'))
+ def test_build_liminal(self):
+ liminal_apps_builder.build_liminal_apps(
+ os.path.join(os.path.dirname(__file__), '../liminal'))
for image in self.__image_names:
self.docker_client.images.get(image)
diff --git a/tests/runners/airflow/dag/test_rainbow_dags.py b/tests/runners/airflow/dag/test_liminal_dags.py
similarity index 87%
rename from tests/runners/airflow/dag/test_rainbow_dags.py
rename to tests/runners/airflow/dag/test_liminal_dags.py
index 5ffdf07..1069278 100644
--- a/tests/runners/airflow/dag/test_rainbow_dags.py
+++ b/tests/runners/airflow/dag/test_liminal_dags.py
@@ -2,8 +2,8 @@ import os
import unittest
from unittest import TestCase
-from rainbow.runners.airflow.dag import rainbow_dags
-from rainbow.runners.airflow.operators.job_status_operator import JobEndOperator, JobStartOperator
+from liminal.runners.airflow.dag import liminal_dags
+from liminal.runners.airflow.operators.job_status_operator import JobEndOperator, JobStartOperator
class Test(TestCase):
@@ -42,8 +42,8 @@ class Test(TestCase):
@staticmethod
def get_register_dags():
- base_path = os.path.join(os.path.dirname(__file__), '../rainbow')
- return rainbow_dags.register_dags(base_path)
+ base_path = os.path.join(os.path.dirname(__file__), '../liminal')
+ return liminal_dags.register_dags(base_path)
if __name__ == '__main__':
diff --git a/rainbow/build/image/__init__.py b/tests/runners/airflow/liminal/__init__.py
similarity index 100%
rename from rainbow/build/image/__init__.py
rename to tests/runners/airflow/liminal/__init__.py
diff --git a/rainbow/build/__init__.py b/tests/runners/airflow/liminal/helloworld/__init__.py
similarity index 100%
rename from rainbow/build/__init__.py
rename to tests/runners/airflow/liminal/helloworld/__init__.py
diff --git a/tests/runners/airflow/rainbow/helloworld/hello_world.py b/tests/runners/airflow/liminal/helloworld/hello_world.py
similarity index 90%
rename from tests/runners/airflow/rainbow/helloworld/hello_world.py
rename to tests/runners/airflow/liminal/helloworld/hello_world.py
index 95f4e73..0b92ae0 100644
--- a/tests/runners/airflow/rainbow/helloworld/hello_world.py
+++ b/tests/runners/airflow/liminal/helloworld/hello_world.py
@@ -20,8 +20,8 @@ import os
print('Hello world!\n')
-with open('/rainbow_input.json') as file:
- print(f'rainbow_input.json contents = {json.loads(file.readline())}')
+with open('/liminal_input.json') as file:
+ print(f'liminal_input.json contents = {json.loads(file.readline())}')
os.makedirs('/mnt/vol1/', exist_ok=True)
diff --git a/tests/runners/airflow/rainbow/rainbow.yml b/tests/runners/airflow/liminal/liminal.yml
similarity index 100%
rename from tests/runners/airflow/rainbow/rainbow.yml
rename to tests/runners/airflow/liminal/liminal.yml
diff --git a/rainbow/__init__.py b/tests/runners/airflow/liminal/myserver/__init__.py
similarity index 100%
rename from rainbow/__init__.py
rename to tests/runners/airflow/liminal/myserver/__init__.py
diff --git a/tests/runners/airflow/rainbow/myserver/my_server.py b/tests/runners/airflow/liminal/myserver/my_server.py
similarity index 100%
rename from tests/runners/airflow/rainbow/myserver/my_server.py
rename to tests/runners/airflow/liminal/myserver/my_server.py
diff --git a/tests/runners/airflow/rainbow/pip.conf b/tests/runners/airflow/liminal/pip.conf
similarity index 100%
rename from tests/runners/airflow/rainbow/pip.conf
rename to tests/runners/airflow/liminal/pip.conf
diff --git a/tests/runners/airflow/rainbow/requirements.txt b/tests/runners/airflow/liminal/requirements.txt
similarity index 100%
rename from tests/runners/airflow/rainbow/requirements.txt
rename to tests/runners/airflow/liminal/requirements.txt
diff --git a/tests/runners/airflow/tasks/defaults/test_job_end.py b/tests/runners/airflow/tasks/defaults/test_job_end.py
index 9a2c398..a5606f4 100644
--- a/tests/runners/airflow/tasks/defaults/test_job_end.py
+++ b/tests/runners/airflow/tasks/defaults/test_job_end.py
@@ -19,7 +19,7 @@
import unittest
from unittest import TestCase
-from rainbow.runners.airflow.tasks.defaults import job_end
+from liminal.runners.airflow.tasks.defaults import job_end
from tests.util import dag_test_utils
diff --git a/tests/runners/airflow/tasks/defaults/test_job_start.py b/tests/runners/airflow/tasks/defaults/test_job_start.py
index d07cf4b..c87f499 100644
--- a/tests/runners/airflow/tasks/defaults/test_job_start.py
+++ b/tests/runners/airflow/tasks/defaults/test_job_start.py
@@ -19,7 +19,7 @@
import unittest
from unittest import TestCase
-from rainbow.runners.airflow.tasks.defaults import job_end, job_start
+from liminal.runners.airflow.tasks.defaults import job_end, job_start
from tests.util import dag_test_utils
diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py
index ac295eb..cb76ece 100644
--- a/tests/runners/airflow/tasks/test_python.py
+++ b/tests/runners/airflow/tasks/test_python.py
@@ -19,9 +19,9 @@
import unittest
from unittest import TestCase
-from rainbow.runners.airflow.operators.kubernetes_pod_operator_with_input_output import \
+from liminal.runners.airflow.operators.kubernetes_pod_operator_with_input_output import \
KubernetesPodOperatorWithInputAndOutput
-from rainbow.runners.airflow.tasks import python
+from liminal.runners.airflow.tasks import python
from tests.util import dag_test_utils
@@ -49,7 +49,7 @@ class TestPythonTask(TestCase):
return {
'task': task_id,
'cmd': 'foo bar',
- 'image': 'rainbow_image',
+ 'image': 'liminal_image',
'source': 'baz',
'input_type': 'my_input_type',
'input_path': 'my_input',
diff --git a/tests/util/test_class_utils.py b/tests/util/test_class_utils.py
index 0deeff6..e0f3d5c 100644
--- a/tests/util/test_class_utils.py
+++ b/tests/util/test_class_utils.py
@@ -1,6 +1,6 @@
from unittest import TestCase
-from rainbow.core.util import class_util
+from liminal.core.util import class_util
from tests.util.test_pkg_1.test_clazz_base import A, Z
from tests.util.test_pkg_1.test_pkg_1_1.test_clazz_child_1 import B
from tests.util.test_pkg_1.test_pkg_1_1.test_clazz_child_2 import C