You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@liminal.apache.org by av...@apache.org on 2021/12/14 14:14:48 UTC

[incubator-liminal] branch master updated: [LIMINAL-86] Enable Black - Python code formatter

This is an automated email from the ASF dual-hosted git repository.

aviemzur pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git


The following commit(s) were added to refs/heads/master by this push:
     new 2947390  [LIMINAL-86] Enable Black - Python code formatter
2947390 is described below

commit 2947390aaf83864354e8c979bac4e44d364b2cbe
Author: ChethanUK <ch...@gmail.com>
AuthorDate: Tue Dec 14 19:44:21 2021 +0530

    [LIMINAL-86] Enable Black - Python code formatter
---
 .github/PULL_REQUEST_TEMPLATE.md                   |  23 +
 .github/workflows/pre_commits.yml                  |  57 ++
 .../dag_test_utils.py => .pre-commit-config.yaml   |  39 +-
 CONTRIBUTING.md                                    |  49 +-
 README.md                                          |  65 +-
 docs/conf.py                                       |  23 +-
 examples/aws-ml-app-demo/model_store.py            |   4 +-
 examples/aws-ml-app-demo/training.py               |   3 +
 examples/spark-app-demo/k8s/data_cleanup.py        |  13 +-
 examples/spark-app-demo/k8s/model_store.py         |   3 +-
 examples/spark-app-demo/k8s/training.py            |  10 +-
 liminal/build/image/python/python.py               |   1 -
 .../image/python_server/liminal_python_server.py   |  12 +-
 liminal/build/image/python_server/python_server.py |   6 +-
 liminal/build/image_builder.py                     |   9 +-
 liminal/build/liminal_apps_builder.py              |   7 +-
 liminal/build/python.py                            |  14 +-
 liminal/core/config/config.py                      |  38 +-
 liminal/core/config/defaults/default_configs.py    |  91 ++-
 liminal/core/environment.py                        |   7 +-
 liminal/kubernetes/volume_util.py                  |  54 +-
 liminal/logging/logging_setup.py                   |   9 +-
 liminal/runners/airflow/__init__.py                |   8 +-
 liminal/runners/airflow/dag/__init__.py            |   5 +-
 .../runners/airflow/dag/liminal_register_dags.py   |  27 +-
 liminal/runners/airflow/executors/emr.py           |  20 +-
 liminal/runners/airflow/executors/kubernetes.py    |  58 +-
 liminal/runners/airflow/model/executor.py          |  11 +-
 liminal/runners/airflow/model/task.py              |   5 +-
 .../runners/airflow/operators/cloudformation.py    |  83 ++-
 .../airflow/operators/job_status_operator.py       |  61 +-
 .../operators/operator_with_variable_resolving.py  |  94 ++-
 liminal/runners/airflow/tasks/containerable.py     |  15 +-
 .../airflow/tasks/create_cloudformation_stack.py   |  36 +-
 .../airflow/tasks/delete_cloudformation_stack.py   |  21 +-
 liminal/runners/airflow/tasks/job_end.py           |  12 +-
 liminal/runners/airflow/tasks/job_start.py         |  10 +-
 liminal/runners/airflow/tasks/python.py            |   5 +-
 liminal/runners/airflow/tasks/spark.py             |  18 +-
 .../airflow/dag/__init__.py => pyproject.toml      |  26 +-
 requirements.txt                                   |   1 +
 scripts/webserver_config.py                        |   2 +-
 setup.py                                           |  11 +-
 .../defaults/test_apply_variables_substitution.py  |  36 +-
 .../defaults/test_defaults_pipeline_config.py      |  73 +--
 .../defaults/test_defaults_service_config.py       |  48 +-
 .../config/defaults/test_defaults_tasks_config.py  | 177 ++----
 tests/liminal/core/config/test_config.py           | 657 +++++++++++----------
 tests/liminal/core/util/test_dict_utils.py         | 134 ++---
 tests/liminal/kubernetes/test_volume_util.py       |  19 +-
 .../python/test_python_server_image_builder.py     |  38 +-
 .../build/python/test_python_image_builder.py      |  37 +-
 .../build/spark/test_spark_image_builder.py        |  24 +-
 .../airflow/build/test_liminal_apps_builder.py     |   9 +-
 tests/runners/airflow/executors/test_emr.py        |  38 +-
 .../airflow/liminal/write_outputs/write_outputs.py |   8 +-
 .../test_operator_with_variable_resolving.py       | 153 +++--
 .../tasks/test_create_cloudformation_stack.py      |  63 +-
 .../tasks/test_delete_cloudformation_stack.py      |  35 +-
 tests/runners/airflow/tasks/test_job_end.py        |  32 +-
 tests/runners/airflow/tasks/test_job_start.py      |  26 +-
 tests/runners/airflow/tasks/test_python.py         | 116 ++--
 tests/runners/airflow/tasks/test_spark_task.py     | 150 +++--
 .../apps/test_spark_app/wordcount/wordcount.py     |  12 +-
 tests/test_licenses.py                             |  34 +-
 tests/util/dag_test_utils.py                       |   5 +-
 tests/util/test_class_utils.py                     |   2 +-
 67 files changed, 1404 insertions(+), 1588 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000..4e9c4ef
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,23 @@
+## Changes Title (replace this with a logical title for your changes)
+
+*Issue #, if available:*
+
+### *Description of changes:*
+
+Replace this with the PR description (mention the changes you have made, why you have made them, provide some background and any references to the provider documentation if needed, etc.).
+
+For more information on contributing, please see [Contributing](../CONTRIBUTING.md)
+section of our documentation.
+
+### Status
+
+Replace this: describe the PR status. Examples:
+
+- WIP - work in progress
+- DONE - ready for review
+
+### Checklist (tick everything that applies)
+
+- [ ] [PreCommitChecks - Code linting](../CONTRIBUTING.md#InstallRunPreCommit) (required)
+- [ ] [Tests](../CONTRIBUTING.md#RunningTests)
+
diff --git a/.github/workflows/pre_commits.yml b/.github/workflows/pre_commits.yml
new file mode 100644
index 0000000..42d064c
--- /dev/null
+++ b/.github/workflows/pre_commits.yml
@@ -0,0 +1,57 @@
+name: PreCommitChecks
+
+on:
+  push:
+    branches:
+      - master
+      # - '!release*'
+  pull_request:
+    branches:
+      - master
+      # - '!release*'
+jobs:
+  linting:
+    name: "Run pre-commit hooks on py3.6"
+    runs-on: ubuntu-latest
+    steps:
+      #----------------------------------------------
+      # Checkout, SetUp Python, Load Cache and Run PreCommitChecks
+      #----------------------------------------------
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      # Install Python
+      - uses: actions/setup-python@v2
+        with:
+            python-version: 3.6
+
+      - name: Check Python ${{ matrix.python-version }} version
+        run: python -V
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: ${{ runner.os }}-pip
+
+      - name: Install python requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pre-commit
+
+      # load cached venv if cache exists
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}
+
+      - run: pre-commit install
+      - name: Run pre-commit hooks on all the files
+        run: pre-commit run --all-files --show-diff-on-failure --color always --verbose
+      # - uses: pre-commit/action@v2.0.3
+      #   with:
+      #     token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/tests/util/dag_test_utils.py b/.pre-commit-config.yaml
similarity index 60%
copy from tests/util/dag_test_utils.py
copy to .pre-commit-config.yaml
index b1fbcab..bc7cac5 100644
--- a/tests/util/dag_test_utils.py
+++ b/.pre-commit-config.yaml
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,19 +14,27 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+---
+minimum_pre_commit_version: "2.16.0"
+exclude: >
+  (?x)^(
+      .+/.venv/.+|.+/dist/.+|.+/.autovenv|.+/docs/|.github
+  )$
+fail_fast: true
+default_language_version:
+  python: python3
+default_stages:
+  - prepare-commit-msg
+  - commit
+  # - push
 
-
-from datetime import datetime
-
-from airflow import DAG
-
-
-def create_dag():
-    """
-    Test util to create a basic DAG for testing.
-    """
-
-    return DAG(
-        dag_id='test_dag',
-        default_args={'start_date': datetime(1970, 1, 1)}
-    )
+repos:
+  - repo: https://github.com/psf/black
+    rev: 21.12b0
+    hooks:
+      - id: black
+        args: [ --safe, --quiet ]
+        files: \.pyi?$
+        exclude: .github/
+        # override until resolved: https://github.com/psf/black/issues/402
+        types: [ ]
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 854e260..9683595 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -24,10 +24,10 @@ Apache Liminal use Sphinx and readthedocs.org to create and publish documentatio
 ## Basic setup
 Here is what you need to do in order to work and create docs locally
 
-## Installing sphinx and documentation 
+## Installing sphinx and documentation
 
 More details  about sphinx and readthedocs can be found [here:](https://docs.readthedocs.io/en/stable/intro/getting-started-with-sphinx.html)
- 
+
 Here is what you need to do:
 ```
 pip install sphinx
@@ -44,15 +44,58 @@ pip install sphinx-autobuild
 cd docs
 sphinx-autobuild source/ build/html/
 ```
-Now open a browser on 
+Now open a browser on
 http://localhost:8000/
 Your docs will automatically update
 
 ## Rebuilding the pythondoc liminal code documentation
+
 ```
 cd docs
 sphinx-apidoc -o source/ ../liminal
 ```
 
+## RunningTests
+
+When doing local development and running Liminal unit-tests, make sure to set LIMINAL_STAND_ALONE_MODE=True
+
+1. Setup Minikube
+2.Install python requirements: `pip install -r requirements.txt`
+3. Run tests: `./run_tests.sh`
+
+## InstallRunPreCommit
+
+[pre-commit](https://pre-commit.com/) is used to install Python code linting and formatting tools:
+
+### Getting started
+
+**Requires `python >=3.6`, `pre-commit>=1.14` and a `git` repository**
+
+1. Run `pip install pre-commit` or `pip install  -r requirements.txt`
+2. Install the hooks:
+  Simple install without post hooks: `pre-commit install`
+
+  OR
+  Install the hooks: `pre-commit install --install-hooks`
+
+  Optional:
+  Install the post commit hooks: `pre-commit install --hook-type post-commit`
+
+1. To run pre commit hooks:
+   1. Either run `git commit` the new configuration files
+
+   1. Run `pre-commit run -a` to lint and format your entire project
+
+Now on every commit, `pre-commit` will use a git hook to run the tools.
+**Warning: the first commit will take some time because the tools are being installed by
+`pre-commit`**
+
+`pre-commit run -a` to lint and format your entire project
+
+### Resolving failed commits
+
+* If `black` fail, they have reformatted your code. `git add` and `git commit` the changes.
 
+Example:
 
+![Fixing black failed commits](https://user-images.githubusercontent.com/16241795/146011210-7bc11b24-2033-43f7-8150-5ece4fe7bfea.png)
diff --git a/README.md b/README.md
index c69f564..637ec46 100644
--- a/README.md
+++ b/README.md
@@ -35,15 +35,19 @@ Using simple YAML configuration, create your own schedule data pipelines (a sequ
 perform), application servers,  and more.
 
 ## Getting Started
+
 A simple getting stated guide for Liminal can be found [here](docs/getting_started.md)
 
 ## Apache Liminal Documentation
+
 Full documentation of Apache Liminal can be found [here](docs/liminal)
 
 ## High Level Architecture
+
 High level architecture documentation can be found [here](docs/architecture.md)
 
 ## Example YAML config file
+
 ```yaml
 ---
 name: MyLiminalStack
@@ -104,13 +108,16 @@ services:
     image: my_server_image
 ```
 
-
 # Installation
+
 1. Install this repository (HEAD)
+
 ```bash
    pip install git+https://github.com/apache/incubator-liminal.git
 ```
+
 2. Optional: set LIMINAL_HOME to path of your choice (if not set, will default to ~/liminal_home)
+
 ```bash
 echo 'export LIMINAL_HOME=</path/to/some/folder>' >> ~/.bash_profile && source ~/.bash_profile
 ```
@@ -119,11 +126,11 @@ echo 'export LIMINAL_HOME=</path/to/some/folder>' >> ~/.bash_profile && source ~
 
 This involves at minimum creating a single file called liminal.yml as in the example above.
 
-If your pipeline requires custom python code to implement tasks, they should be organized 
+If your pipeline requires custom python code to implement tasks, they should be organized
 [like this](https://github.com/apache/incubator-liminal/tree/master/tests/runners/airflow/liminal)
 
-If your pipeline  introduces imports of external packages which are not already a part 
-of the liminal framework (i.e. you had to pip install them yourself), you need to also provide 
+If your pipeline  introduces imports of external packages which are not already a part
+of the liminal framework (i.e. you had to pip install them yourself), you need to also provide
 a requirements.txt in the root of your project.
 
 # Testing the pipeline locally
@@ -131,22 +138,24 @@ a requirements.txt in the root of your project.
 When your pipeline code is ready, you can test it by running it locally on your machine.
 
 1. Ensure you have The Docker engine running locally, and enable a local Kubernetes cluster:
-![Kubernetes configured](https://raw.githubusercontent.com/apache/incubator-liminal/master/images/k8s_running.png)
 
-And allocate it at least 3 CPUs (under "Resources" in the Docker preference UI).
+  ![Kubernetes configured](https://raw.githubusercontent.com/apache/incubator-liminal/master/images/k8s_running.png)
+
+  And allocate it at least 3 CPUs (under "Resources" in the Docker preference UI).
+
+  If you want to execute your pipeline on a remote kubernetes cluster, make sure the cluster is configured using:
+
+  ```bash
+  kubectl config set-context <your remote kubernetes cluster>
+  ```
 
-If you want to execute your pipeline on a remote kubernetes cluster, make sure the cluster is configured
-using :
-```bash
-kubectl config set-context <your remote kubernetes cluster>
-``` 
 2. Build the docker images used by your pipeline.
 
-In the example pipeline above, you can see that tasks and services have an "image" field - such as 
-"my_static_input_task_image". This means that the task is executed inside a docker container, and the docker container 
+In the example pipeline above, you can see that tasks and services have an "image" field - such as
+"my_static_input_task_image". This means that the task is executed inside a docker container, and the docker container
 is created from a docker image where various code and libraries are installed.
 
-You can take a look at what the build process looks like, e.g. 
+You can take a look at what the build process looks like, e.g.
 [here](https://github.com/apache/incubator-liminal/tree/master/liminal/build/image/python)
 
 In order for the images to be available for your pipeline, you'll need to build them locally:
@@ -161,34 +170,41 @@ You'll see that a number of outputs indicating various docker images built.
 3. Create a kubernetes local volume \
 In case your Yaml includes working with [volumes](https://github.com/apache/incubator-liminal/blob/6253f8b2c9dc244af032979ec6d462dc3e07e170/docs/getting_started.md#mounted-volumes)
 please first run the following command:
+
 ```bash
-cd </path/to/your/liminal/code> 
+cd </path/to/your/liminal/code>
 liminal create
 ```
 
 4. Deploy the pipeline:
+
 ```bash
-cd </path/to/your/liminal/code> 
+cd </path/to/your/liminal/code>
 liminal deploy
 ```
-Note: after upgrading liminal, it's recommended to issue the command 
+
+Note: after upgrading liminal, it's recommended to issue the command
+
 ```bash
 liminal deploy --clean
-``` 
+```
 
 This will rebuild the airlfow docker containers from scratch with a fresh version of liminal, ensuring consistency.
 
 5. Start the server
+
 ```bash
 liminal start
 ```
 
 6. Stop the server
+
 ```bash
 liminal stop
 ```
 
 7. Display the server logs
+
 ```bash
 liminal logs --follow/--tail
 
@@ -205,8 +221,9 @@ liminal logs --follow
 The pipeline is scheduled to run according to the ```json schedule: 0 * 1 * *``` field in the .yml file you provided.
 
 10. To manually activate your pipeline:
-Click your pipeline and then click "trigger DAG"
-Click "Graph view"
+
+- Click your pipeline and then click "trigger DAG"
+- Click "Graph view"
 You should see the steps in your pipeline getting executed in "real time" by clicking "Refresh" periodically.
 
 ![Pipeline activation](https://raw.githubusercontent.com/apache/incubator-liminal/master/images/airflow_trigger.png)
@@ -215,5 +232,13 @@ You should see the steps in your pipeline getting executed in "real time" by cli
 
 More information on contributing can be found [here](CONTRIBUTING.md)
 
+# Community
+
+The Liminal community holds a public call every Monday
+
+- [Liminal Community Calendar](https://calendar.google.com/calendar/u/0/r?cid=jom1i20emghura6s6ookhe2skk@group.calendar.google.com)
+- [Dev-Mailing-List](https://lists.apache.org/list.html?dev@liminal.apache.org)
+
 ## Running Tests (for contributors)
+
 When doing local development and running Liminal unit-tests, make sure to set LIMINAL_STAND_ALONE_MODE=True
diff --git a/docs/conf.py b/docs/conf.py
index 5eb6888..6008c83 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -129,15 +129,12 @@ latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -147,18 +144,14 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'ApchaeLiminal.tex', u'Apchae Liminal Documentation',
-     u'Apache Software Foundation', 'manual'),
+    (master_doc, 'ApchaeLiminal.tex', u'Apchae Liminal Documentation', u'Apache Software Foundation', 'manual'),
 ]
 
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'apchaeliminal', u'Apchae Liminal Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, 'apchaeliminal', u'Apchae Liminal Documentation', [author], 1)]
 
 # -- Options for Texinfo output ----------------------------------------------
 
@@ -166,9 +159,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'ApchaeLiminal', u'Apchae Liminal Documentation',
-     author, 'ApchaeLiminal', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        'ApchaeLiminal',
+        u'Apchae Liminal Documentation',
+        author,
+        'ApchaeLiminal',
+        'One line description of project.',
+        'Miscellaneous',
+    ),
 ]
 
 # -- Options for Epub output -------------------------------------------------
diff --git a/examples/aws-ml-app-demo/model_store.py b/examples/aws-ml-app-demo/model_store.py
index 191861f..1f23270 100644
--- a/examples/aws-ml-app-demo/model_store.py
+++ b/examples/aws-ml-app-demo/model_store.py
@@ -28,8 +28,8 @@ CANDIDATE = 'candidate'
 
 _ONE_HOUR = 60 * 60
 
-class ModelStore:
 
+class ModelStore:
     def __init__(self, env):
         self.env = env
         self._latest_model = None
@@ -50,7 +50,7 @@ class ModelStore:
         pickle.dump(model, open(f'{path}/{key}', "wb"))
 
     def _download_latest_model(self):
-        objects = (glob.glob(f'{MOUNT_PATH}/{self.env}/**/*'))
+        objects = glob.glob(f'{MOUNT_PATH}/{self.env}/**/*')
         models = list(reversed(sorted([obj for obj in objects if obj.endswith('.p')])))
         latest_key = models[0]
         version = latest_key.rsplit('/')[-2]
diff --git a/examples/aws-ml-app-demo/training.py b/examples/aws-ml-app-demo/training.py
index 505f814..d1fcd8c 100644
--- a/examples/aws-ml-app-demo/training.py
+++ b/examples/aws-ml-app-demo/training.py
@@ -28,6 +28,7 @@ from sklearn.linear_model import LogisticRegression
 _CANDIDATE_MODEL_STORE = ModelStore(model_store.CANDIDATE)
 _PRODUCTION_MODEL_STORE = ModelStore(model_store.PRODUCTION)
 
+
 def train_model():
     iris = datasets.load_iris()
 
@@ -42,6 +43,7 @@ def train_model():
     print(f'Saving model with version {version} to candidate model store.')
     _CANDIDATE_MODEL_STORE.save_model(model, version)
 
+
 def validate_model():
     model, version = _CANDIDATE_MODEL_STORE.load_latest_model()
     print(f'Validating model with version {version} to candidate model store.')
@@ -50,6 +52,7 @@ def validate_model():
     print(f'Deploying model with version {version} to production model store.')
     _PRODUCTION_MODEL_STORE.save_model(model, version)
 
+
 if __name__ == '__main__':
     cmd = sys.argv[1]
     if cmd == 'train':
diff --git a/examples/spark-app-demo/k8s/data_cleanup.py b/examples/spark-app-demo/k8s/data_cleanup.py
index 0fd5617..c78d0aa 100644
--- a/examples/spark-app-demo/k8s/data_cleanup.py
+++ b/examples/spark-app-demo/k8s/data_cleanup.py
@@ -28,16 +28,14 @@ from pyspark.sql import SparkSession
 def transform(data):
     columns_to_scale = data.columns[:-1]
     vectorizer = VectorAssembler(inputCols=columns_to_scale, outputCol="features")
-    scaler = StandardScaler(inputCol="features", outputCol="scaled_features",
-                            withStd=True, withMean=True)
+    scaler = StandardScaler(inputCol="features", outputCol="scaled_features", withStd=True, withMean=True)
     labeler = StringIndexer(inputCol=data.columns[-1], outputCol='label')
     pipeline = Pipeline(stages=[vectorizer, scaler, labeler])
     fitted = pipeline.fit(data)
     transformed = fitted.transform(data)
 
-    result = (
-        transformed.withColumn("feature_arr", vector_to_array("scaled_features"))
-        .select([F.col("feature_arr")[i].alias(columns_to_scale[i]) for i in range(len(columns_to_scale))] + ['label'])
+    result = transformed.withColumn("feature_arr", vector_to_array("scaled_features")).select(
+        [F.col("feature_arr")[i].alias(columns_to_scale[i]) for i in range(len(columns_to_scale))] + ['label']
     )
 
     return result
@@ -52,10 +50,7 @@ def load(data, output_uri):
 
 
 def data_pipeline(input_uri, output_uri):
-    spark = SparkSession \
-        .builder \
-        .appName("Prepare Iris Data") \
-        .getOrCreate()
+    spark = SparkSession.builder.appName("Prepare Iris Data").getOrCreate()
 
     input = extract(spark, input_uri)
     data = transform(input)
diff --git a/examples/spark-app-demo/k8s/model_store.py b/examples/spark-app-demo/k8s/model_store.py
index f2c8509..1f23270 100644
--- a/examples/spark-app-demo/k8s/model_store.py
+++ b/examples/spark-app-demo/k8s/model_store.py
@@ -30,7 +30,6 @@ _ONE_HOUR = 60 * 60
 
 
 class ModelStore:
-
     def __init__(self, env):
         self.env = env
         self._latest_model = None
@@ -51,7 +50,7 @@ class ModelStore:
         pickle.dump(model, open(f'{path}/{key}', "wb"))
 
     def _download_latest_model(self):
-        objects = (glob.glob(f'{MOUNT_PATH}/{self.env}/**/*'))
+        objects = glob.glob(f'{MOUNT_PATH}/{self.env}/**/*')
         models = list(reversed(sorted([obj for obj in objects if obj.endswith('.p')])))
         latest_key = models[0]
         version = latest_key.rsplit('/')[-2]
diff --git a/examples/spark-app-demo/k8s/training.py b/examples/spark-app-demo/k8s/training.py
index 57c4e6c..2857c95 100644
--- a/examples/spark-app-demo/k8s/training.py
+++ b/examples/spark-app-demo/k8s/training.py
@@ -35,13 +35,15 @@ import numpy as np
 import csv
 import argparse
 
+
 def load_iris_from_csv_file(f):
     df = pd.read_csv(f, header=0).reset_index(drop=True)
-    types = {col:np.float64 for col in df.columns[:-1]}
+    types = {col: np.float64 for col in df.columns[:-1]}
     types['label'] = np.int32
     df = df.astype(types)
     return df
 
+
 def get_dataset(d):
     print("searching for csv files in {}".format(d))
     for root, dirs, files in os.walk(d):
@@ -51,6 +53,7 @@ def get_dataset(d):
                 return os.path.join(d, file)
     return None
 
+
 def load_and_split(input_uri):
     csv_file = get_dataset(input_uri)
     if csv_file:
@@ -59,6 +62,7 @@ def load_and_split(input_uri):
     iris = load_iris_from_csv_file(csv_file)
     return train_test_split(iris, test_size=0.2, random_state=8)
 
+
 def train_model(input_uri):
     train, test = load_and_split(input_uri)
     y = train.pop("label")
@@ -79,7 +83,7 @@ def train_model(input_uri):
 def validate_model(input_uri):
     model, version = _CANDIDATE_MODEL_STORE.load_latest_model()
     print(f'Validating model with version {version} to candidate model store.')
-    if not isinstance(model.predict([[1,1,1,1]]), np.ndarray):
+    if not isinstance(model.predict([[1, 1, 1, 1]]), np.ndarray):
         raise ValueError('Invalid model')
     train, test = load_and_split(input_uri)
     y = test.pop("label")
@@ -100,4 +104,4 @@ if __name__ == '__main__':
     if args.action == 'train':
         train_model(args.input_uri)
     else:
-        validate_model(args.input_uri)
\ No newline at end of file
+        validate_model(args.input_uri)
diff --git a/liminal/build/image/python/python.py b/liminal/build/image/python/python.py
index b318479..b9dc557 100644
--- a/liminal/build/image/python/python.py
+++ b/liminal/build/image/python/python.py
@@ -22,7 +22,6 @@ from liminal.build.python import BasePythonImageBuilder
 
 
 class PythonImageBuilder(BasePythonImageBuilder):
-
     def __init__(self, config, base_path, relative_source_path, tag):
         super().__init__(config, base_path, relative_source_path, tag)
 
diff --git a/liminal/build/image/python_server/liminal_python_server.py b/liminal/build/image/python_server/liminal_python_server.py
index db79308..2099fd3 100644
--- a/liminal/build/image/python_server/liminal_python_server.py
+++ b/liminal/build/image/python_server/liminal_python_server.py
@@ -38,14 +38,15 @@ if __name__ == '__main__':
     with open('service.yml') as stream:
         config = yaml.safe_load(stream)
 
-    endpoints = dict([
-        (endpoint_config['endpoint'][1:], __get_endpoint_function(endpoint_config))
-        for endpoint_config in config['endpoints']
-    ])
+    endpoints = dict(
+        [
+            (endpoint_config['endpoint'][1:], __get_endpoint_function(endpoint_config))
+            for endpoint_config in config['endpoints']
+        ]
+    )
 
     blueprint = Blueprint('liminal_python_server_blueprint', __name__)
 
-
     @blueprint.route('/', defaults={'endpoint': ''}, methods=('GET', 'POST'))
     @blueprint.route('/<endpoint>', methods=('GET', 'POST'))
     def show(endpoint):
@@ -54,7 +55,6 @@ if __name__ == '__main__':
         else:
             return 'Page not found.', 404
 
-
     app = Flask(__name__)
     app.register_blueprint(blueprint)
 
diff --git a/liminal/build/image/python_server/python_server.py b/liminal/build/image/python_server/python_server.py
index 25fcaa3..77be30f 100644
--- a/liminal/build/image/python_server/python_server.py
+++ b/liminal/build/image/python_server/python_server.py
@@ -24,7 +24,6 @@ from liminal.build.python import BasePythonImageBuilder
 
 
 class PythonServerImageBuilder(BasePythonImageBuilder):
-
     def __init__(self, config, base_path, relative_source_path, tag):
         super().__init__(config, base_path, relative_source_path, tag)
 
@@ -36,9 +35,8 @@ class PythonServerImageBuilder(BasePythonImageBuilder):
     def _additional_files_from_paths():
         return [
             os.path.join(os.path.dirname(__file__), 'liminal_python_server.py'),
-            os.path.join(os.path.dirname(__file__), 'python_server_requirements.txt')
+            os.path.join(os.path.dirname(__file__), 'python_server_requirements.txt'),
         ]
 
     def _additional_files_from_filename_content_pairs(self):
-        return super()._additional_files_from_filename_content_pairs() + \
-               [('service.yml', yaml.safe_dump(self.config))]
+        return super()._additional_files_from_filename_content_pairs() + [('service.yml', yaml.safe_dump(self.config))]
diff --git a/liminal/build/image_builder.py b/liminal/build/image_builder.py
index cfc423f..428d749 100644
--- a/liminal/build/image_builder.py
+++ b/liminal/build/image_builder.py
@@ -58,8 +58,7 @@ class ImageBuilder:
             no_cache = '--no-cache=true'
 
         docker = 'docker' if shutil.which('docker') is not None else '/usr/local/bin/docker'
-        docker_build_command = f'{docker} build {no_cache} ' + \
-                               f'--tag {self.tag} '
+        docker_build_command = f'{docker} build {no_cache} ' + f'--tag {self.tag} '
 
         docker_build_command += f'--progress=plain {self._build_flags()} '
 
@@ -72,9 +71,9 @@ class ImageBuilder:
 
         docker_build_out = ''
         try:
-            docker_build_out = subprocess.check_output(docker_build_command,
-                                                       shell=True, stderr=subprocess.STDOUT,
-                                                       timeout=960)
+            docker_build_out = subprocess.check_output(
+                docker_build_command, shell=True, stderr=subprocess.STDOUT, timeout=960
+            )
         except subprocess.CalledProcessError as e:
             docker_build_out = e.output
             raise e
diff --git a/liminal/build/liminal_apps_builder.py b/liminal/build/liminal_apps_builder.py
index 1b3fb8b..49ebf21 100644
--- a/liminal/build/liminal_apps_builder.py
+++ b/liminal/build/liminal_apps_builder.py
@@ -53,7 +53,8 @@ def __build_image(base_path, builder_config, builder):
         config=builder_config,
         base_path=base_path,
         relative_source_path=builder_config['source'],
-        tag=builder_config['image'])
+        tag=builder_config['image'],
+    )
     builder_instance.build()
 
 
@@ -67,9 +68,7 @@ logging.info(f'Loading image builder implementations..')
 image_builders_package = 'liminal.build.image'
 # user_image_builders_package = 'TODO: user_image_builders_package'
 
-image_builder_types = class_util.find_subclasses_in_packages(
-    [image_builders_package],
-    ImageBuilder)
+image_builder_types = class_util.find_subclasses_in_packages([image_builders_package], ImageBuilder)
 
 logging.info(f'Finished loading image builder implementations: {image_builder_types}')
 logging.info(f'Loading service image builder implementations..')
diff --git a/liminal/build/python.py b/liminal/build/python.py
index e42b63b..bb20be8 100644
--- a/liminal/build/python.py
+++ b/liminal/build/python.py
@@ -47,9 +47,11 @@ class PythonImageVersions:
         else:
             python_version = str(python_version)
         if python_version[:3] not in self.supported_versions:
-            raise ValueError(f'liminal supports the following python versions: '
-                             f'{self.supported_versions} but {python_version} '
-                             f'were passed')
+            raise ValueError(
+                f'liminal supports the following python versions: '
+                f'{self.supported_versions} but {python_version} '
+                f'were passed'
+            )
         return f'python:{python_version}-slim'
 
 
@@ -61,8 +63,7 @@ class BasePythonImageBuilder(ImageBuilder):
     __PIP_CONF = 'pip_conf'
     __PYTHON_VERSION = 'python_version'
 
-    def __init__(self, config, base_path, relative_source_path, tag,
-                 base_image=PythonImageVersions()):
+    def __init__(self, config, base_path, relative_source_path, tag, base_image=PythonImageVersions()):
         super().__init__(config, base_path, relative_source_path, tag)
         self._base_image = base_image
 
@@ -92,8 +93,7 @@ class BasePythonImageBuilder(ImageBuilder):
 
         if self.__PIP_CONF in self.config:
             new_data = '# syntax = docker/dockerfile:1.0-experimental\n' + data
-            new_data = new_data.replace('{{mount}}',
-                                        '--mount=type=secret,id=pip_config,dst=/etc/pip.conf \\\n')
+            new_data = new_data.replace('{{mount}}', '--mount=type=secret,id=pip_config,dst=/etc/pip.conf \\\n')
         else:
             new_data = new_data.replace('{{mount}} ', '')
 
diff --git a/liminal/core/config/config.py b/liminal/core/config/config.py
index 7af3633..a559278 100644
--- a/liminal/core/config/config.py
+++ b/liminal/core/config/config.py
@@ -30,6 +30,7 @@ class ConfigUtil:
     """
     Load and enrich config files under configs_path.
     """
+
     __BASE = 'base'
     __PIPELINES = 'pipelines'
     __SUPER = 'super'
@@ -61,8 +62,7 @@ class ConfigUtil:
         self.config_files = files_util.load(configs_path)
         self.base = base.BASE
         self.loaded_subliminals = []
-        self.snapshot_path = os.path.join(environment.get_airflow_home_dir(),
-                                          '../liminal_config_files')
+        self.snapshot_path = os.path.join(environment.get_airflow_home_dir(), '../liminal_config_files')
 
     def safe_load(self, is_render_variables, soft_merge=False):
         """
@@ -80,8 +80,7 @@ class ConfigUtil:
             # noinspection PyBroadException
             try:
                 superliminal = self.__get_superliminal(subliminal, soft_merge)
-                enriched_config = self.__merge_configs(subliminal, superliminal,
-                                                       is_render_variables, soft_merge)
+                enriched_config = self.__merge_configs(subliminal, superliminal, is_render_variables, soft_merge)
                 enriched_configs.append(enriched_config)
             except Exception:
                 logging.error(f'Failed to load yml {name}')
@@ -98,8 +97,9 @@ class ConfigUtil:
         sub = subliminal.copy()
         supr = superliminal.copy()
 
-        merged_superliminal = self.__merge_configs(supr, self.__get_superliminal(supr, soft_merge),
-                                                   is_render_variables, soft_merge)
+        merged_superliminal = self.__merge_configs(
+            supr, self.__get_superliminal(supr, soft_merge), is_render_variables, soft_merge
+        )
 
         sub[self.__EXECUTORS] = self.__merge_section(sub, merged_superliminal, self.__EXECUTORS)
         sub[self.__IMAGES] = self.__merge_section(sub, merged_superliminal, self.__IMAGES)
@@ -118,8 +118,9 @@ class ConfigUtil:
             else:
                 superliminal = self.__get_config(superliminal_name)
                 if not superliminal:
-                    supr_is_missing_msg = f"superliminal '{superliminal_name}' " + \
-                                          f"is missing from '{self.configs_path}'"
+                    supr_is_missing_msg = (
+                        f"superliminal '{superliminal_name}' " + f"is missing from '{self.configs_path}'"
+                    )
                     if soft_merge:
                         logging.warning(supr_is_missing_msg)
                     else:
@@ -161,25 +162,23 @@ class ConfigUtil:
         super2_pipeline_defaults = super2.get(self.__PIPELINE_DEFAULTS, {}).copy()
 
         super1[self.__PIPELINE_DEFAULTS] = super1_pipeline_defaults
-        super1[self.__PIPELINE_DEFAULTS][self.__BEFORE_TASKS] = \
-            super2_pipeline_defaults.pop(self.__BEFORE_TASKS, []) + super1_pipeline_defaults.pop(
-                self.__BEFORE_TASKS, [])
+        super1[self.__PIPELINE_DEFAULTS][self.__BEFORE_TASKS] = super2_pipeline_defaults.pop(
+            self.__BEFORE_TASKS, []
+        ) + super1_pipeline_defaults.pop(self.__BEFORE_TASKS, [])
 
         super2[self.__PIPELINE_DEFAULTS] = super2_pipeline_defaults
-        super1[self.__PIPELINE_DEFAULTS][self.__AFTER_TASKS] = \
-            super1_pipeline_defaults.pop(self.__AFTER_TASKS, []) + super2_pipeline_defaults.pop(
-                self.__AFTER_TASKS, [])
+        super1[self.__PIPELINE_DEFAULTS][self.__AFTER_TASKS] = super1_pipeline_defaults.pop(
+            self.__AFTER_TASKS, []
+        ) + super2_pipeline_defaults.pop(self.__AFTER_TASKS, [])
 
         # merge supers tasks
         return dict_util.merge_dicts(super1, super2, True)
 
     def snapshot_final_liminal_configs(self):
-        files_util.dump_liminal_configs(liminal_configs=self.loaded_subliminals,
-                                        path=self.snapshot_path)
+        files_util.dump_liminal_configs(liminal_configs=self.loaded_subliminals, path=self.snapshot_path)
 
     def __merge_section(self, subliminal, superliminal, section):
-        return self.__deep_list_keyword_merge(section[:-1], subliminal.get(section, []),
-                                              superliminal.get(section, []))
+        return self.__deep_list_keyword_merge(section[:-1], subliminal.get(section, []), superliminal.get(section, []))
 
     @staticmethod
     def __apply_pipeline_defaults(subliminal, superliminal, pipeline):
@@ -190,5 +189,4 @@ class ConfigUtil:
         subliminal_key_map = {item[unique_key_name]: item for item in subliminal_list_conf}
         superliminal_key_map = {item[unique_key_name]: item for item in superliminal_list_conf}
 
-        return list(dict_util.merge_dicts(superliminal_key_map, subliminal_key_map,
-                                          recursive=True).values())
+        return list(dict_util.merge_dicts(superliminal_key_map, subliminal_key_map, recursive=True).values())
diff --git a/liminal/core/config/defaults/default_configs.py b/liminal/core/config/defaults/default_configs.py
index 25d5001..97a5686 100644
--- a/liminal/core/config/defaults/default_configs.py
+++ b/liminal/core/config/defaults/default_configs.py
@@ -27,21 +27,19 @@ __AFTER_TASKS = "after_tasks"
 
 def apply_variable_substitution(subliminal, superliminal, is_render_variables=False):
     """
-     if is_render_variables is True
-        Replace all {{variable.key}} in subliminal with variable.value variable in
-        subliminal.variables + superliminal.variables
-     else
-        merge subliminal.variables with superliminal.variables without replace
-        placeholders
-     """
+    if is_render_variables is True
+       Replace all {{variable.key}} in subliminal with variable.value variable in
+       subliminal.variables + superliminal.variables
+    else
+       merge subliminal.variables with superliminal.variables without replace
+       placeholders
+    """
     keyword = "variables"
-    merged_variables = dict_util.merge_dicts(subliminal.get(keyword, {}),
-                                             superliminal.get(keyword, {}), True)
+    merged_variables = dict_util.merge_dicts(subliminal.get(keyword, {}), superliminal.get(keyword, {}), True)
     if is_render_variables:
         for k, v in merged_variables.items():
             if isinstance(v, str) or (not isinstance(v, dict) and not isinstance(v, list)):
-                merged_variables[k] = dict_util.replace_placholders_in_string(str(v),
-                                                                              merged_variables)
+                merged_variables[k] = dict_util.replace_placholders_in_string(str(v), merged_variables)
 
         merged_variables = dict_util.replace_placeholders(merged_variables, merged_variables)
         return dict_util.replace_placeholders(subliminal, merged_variables)
@@ -51,21 +49,18 @@ def apply_variable_substitution(subliminal, superliminal, is_render_variables=Fa
 
 
 def apply_service_defaults(subliminal, superliminal):
-    """ Apply defaults services
-        :param subliminal: subliminal config
-        :param superliminal: superliminal config
-        :returns: enriched services with superliminal.service_defaults & subliminal.service_defaults
+    """Apply defaults services
+    :param subliminal: subliminal config
+    :param superliminal: superliminal config
+    :returns: enriched services with superliminal.service_defaults & subliminal.service_defaults
     """
     keyword = "service_defaults"
     superliminal_service_defaults = superliminal.get(keyword, {})
     subliminal_service_defaults = subliminal.get(keyword, {})
 
-    merged_service_defaults = merge_dicts(subliminal_service_defaults,
-                                          superliminal_service_defaults,
-                                          recursive=True)
+    merged_service_defaults = merge_dicts(subliminal_service_defaults, superliminal_service_defaults, recursive=True)
 
-    return [merge_dicts(service, merged_service_defaults, True) for service in
-            subliminal.get(__SERVICES, {})]
+    return [merge_dicts(service, merged_service_defaults, True) for service in subliminal.get(__SERVICES, {})]
 
 
 def apply_pipeline_defaults(subliminal, superliminal, pipeline):
@@ -81,52 +76,48 @@ def apply_pipeline_defaults(subliminal, superliminal, pipeline):
     subliminal_pipe_defaults = subliminal.get(keyword, {}).copy()
     superliminal_before_tasks = superliminal_pipe_defaults.pop(__BEFORE_TASKS, [])
     superliminal_after_tasks = superliminal_pipe_defaults.pop(__AFTER_TASKS, [])
-    merged_pipeline_defaults = merge_dicts(subliminal_pipe_defaults, superliminal_pipe_defaults,
-                                           True)
+    merged_pipeline_defaults = merge_dicts(subliminal_pipe_defaults, superliminal_pipe_defaults, True)
     pipeline = merge_dicts(pipeline, merged_pipeline_defaults, True)
 
-    return apply_task_defaults(subliminal, superliminal, pipeline,
-                               superliminal_before_tasks=superliminal_before_tasks,
-                               superliminal_after_tasks=superliminal_after_tasks)
+    return apply_task_defaults(
+        subliminal,
+        superliminal,
+        pipeline,
+        superliminal_before_tasks=superliminal_before_tasks,
+        superliminal_after_tasks=superliminal_after_tasks,
+    )
 
 
-def apply_task_defaults(subliminal, superliminal, pipeline, superliminal_before_tasks,
-                        superliminal_after_tasks):
+def apply_task_defaults(subliminal, superliminal, pipeline, superliminal_before_tasks, superliminal_after_tasks):
     """Apply defaults task values on given pipeline
-           :param subliminal: subliminal config
-           :param superliminal: superliminal config
-           :param  pipeline: where 'tasks' list can be found it
-           :param  superliminal_before_tasks: superliminal before subtasks list
-           :param  superliminal_after_tasks: superliminal after subtasks list
-
-           :returns: pipeline after enrich with superliminal.tasks and superliminal.task_defaults
-            and subliminal.task_defaults
+    :param subliminal: subliminal config
+    :param superliminal: superliminal config
+    :param  pipeline: where 'tasks' list can be found it
+    :param  superliminal_before_tasks: superliminal before subtasks list
+    :param  superliminal_after_tasks: superliminal after subtasks list
+
+    :returns: pipeline after enrich with superliminal.tasks and superliminal.task_defaults
+     and subliminal.task_defaults
     """
-    pipeline[__TASKS] = __apply_task_defaults(subliminal, superliminal, pipeline.get(__TASKS, []),
-                                              superliminal_before_tasks, superliminal_after_tasks)
+    pipeline[__TASKS] = __apply_task_defaults(
+        subliminal, superliminal, pipeline.get(__TASKS, []), superliminal_before_tasks, superliminal_after_tasks
+    )
 
     return pipeline
 
 
-def __apply_task_defaults(subliminal,
-                          superliminal,
-                          subliminal_tasks,
-                          superliminal_before_tasks,
-                          superliminal_after_tasks):
+def __apply_task_defaults(
+    subliminal, superliminal, subliminal_tasks, superliminal_before_tasks, superliminal_after_tasks
+):
     keyword = "task_defaults"
     subliminal_tasks_defaults = subliminal.get(keyword, {})
     superliminal_tasks_defaults = superliminal.get(keyword, {})
-    merged_task_defaults = merge_dicts(subliminal_tasks_defaults, superliminal_tasks_defaults,
-                                       recursive=True)
+    merged_task_defaults = merge_dicts(subliminal_tasks_defaults, superliminal_tasks_defaults, recursive=True)
 
-    return __enrich_tasks(subliminal_tasks, superliminal_before_tasks, superliminal_after_tasks,
-                          merged_task_defaults)
+    return __enrich_tasks(subliminal_tasks, superliminal_before_tasks, superliminal_after_tasks, merged_task_defaults)
 
 
 def __enrich_tasks(sub_tasks, super_before_tasks, super_after_tasks, task_defaults):
     merged_tasks = super_before_tasks + sub_tasks + super_after_tasks
 
-    return [
-        merge_dicts(task, task_defaults.get(task.get('type', ''), {}), recursive=True)
-        for task in merged_tasks
-    ]
+    return [merge_dicts(task, task_defaults.get(task.get('type', ''), {}), recursive=True) for task in merged_tasks]
diff --git a/liminal/core/environment.py b/liminal/core/environment.py
index 636994c..918ff80 100644
--- a/liminal/core/environment.py
+++ b/liminal/core/environment.py
@@ -47,15 +47,16 @@ def get_dags_dir():
 def get_liminal_version():
     result = os.environ.get(LIMINAL_VERSION_PARAM_NAME, None)
     if not result:
-        output = subprocess.run(['pip freeze | grep \'apache-liminal\''], capture_output=True,
-                                env=os.environ, shell=True)
+        output = subprocess.run(
+            ['pip freeze | grep \'apache-liminal\''], capture_output=True, env=os.environ, shell=True
+        )
         pip_res = output.stdout.decode('UTF-8').strip()
         liminal_home = get_liminal_home()
         whl_files = [file for file in os.listdir(liminal_home) if file.endswith(".whl")]
         if whl_files:
             value = 'file://' + os.path.join(liminal_home, whl_files[0])
         elif ' @ ' in pip_res:
-            value = pip_res[pip_res.index(' @ ') + 3:]
+            value = pip_res[pip_res.index(' @ ') + 3 :]
         else:
             value = pip_res
         logging.info(f'LIMINAL_VERSION not set. Setting it to currently installed version: {value}')
diff --git a/liminal/kubernetes/volume_util.py b/liminal/kubernetes/volume_util.py
index 49db8b1..18de4e9 100644
--- a/liminal/kubernetes/volume_util.py
+++ b/liminal/kubernetes/volume_util.py
@@ -64,16 +64,16 @@ def create_local_volume(conf, namespace='default') -> None:
     _LOG.info(f'Requested volume {name}')
 
     if name not in _LOCAL_VOLUMES:
-        matching_volumes = _kubernetes.list_persistent_volume(
-            field_selector=f'metadata.name={name}'
-        ).to_dict()['items']
+        matching_volumes = _kubernetes.list_persistent_volume(field_selector=f'metadata.name={name}').to_dict()[
+            'items'
+        ]
 
         while len(matching_volumes) == 0:
             _create_local_volume(conf, name)
             sleep(5)
-            matching_volumes = _kubernetes.list_persistent_volume(
-                field_selector=f'metadata.name={name}'
-            ).to_dict()['items']
+            matching_volumes = _kubernetes.list_persistent_volume(field_selector=f'metadata.name={name}').to_dict()[
+                'items'
+            ]
 
         pvc_name = conf.get('claim_name', f'{name}-pvc')
 
@@ -131,9 +131,7 @@ def _list_persistent_volume_claims(name):
 
 
 def _list_persistent_volumes(name):
-    return _kubernetes.list_persistent_volume(
-        field_selector=f'metadata.name={name}'
-    ).to_dict()['items']
+    return _kubernetes.list_persistent_volume(field_selector=f'metadata.name={name}').to_dict()['items']
 
 
 def _create_persistent_volume_claim(pvc_name, volume_name, namespace):
@@ -143,30 +141,21 @@ def _create_persistent_volume_claim(pvc_name, volume_name, namespace):
         'volumeMode': 'Filesystem',
         'storageClassName': 'local-storage',
         'accessModes': ['ReadWriteOnce'],
-        'resources': {
-            'requests': {
-                'storage': '100Gi'
-            }
-        }
+        'resources': {'requests': {'storage': '100Gi'}},
     }
 
     _kubernetes.create_namespaced_persistent_volume_claim(
         namespace,
         V1PersistentVolumeClaim(
-            api_version='v1',
-            kind='PersistentVolumeClaim',
-            metadata={'name': pvc_name},
-            spec=spec
-        )
+            api_version='v1', kind='PersistentVolumeClaim', metadata={'name': pvc_name}, spec=spec
+        ),
     )
 
 
 def _create_local_volume(conf, name):
     _LOG.info(f'Creating persistent volume {name} with spec {conf}')
     spec = {
-        'capacity': {
-            'storage': '100Gi'
-        },
+        'capacity': {'storage': '100Gi'},
         'volumeMode': 'Filesystem',
         'accessModes': ['ReadWriteOnce'],
         'persistentVolumeReclaimPolicy': 'Retain',
@@ -174,29 +163,14 @@ def _create_local_volume(conf, name):
         'nodeAffinity': {
             'required': {
                 'nodeSelectorTerms': [
-                    {
-                        'matchExpressions': [
-                            {
-                                'key': 'kubernetes.io/hostname',
-                                'operator': 'NotIn',
-                                'values': [
-                                    ''
-                                ]
-                            }
-                        ]
-                    }
+                    {'matchExpressions': [{'key': 'kubernetes.io/hostname', 'operator': 'NotIn', 'values': ['']}]}
                 ]
             }
-        }
+        },
     }
 
     spec.update(conf)
 
     _kubernetes.create_persistent_volume(
-        V1PersistentVolume(
-            api_version='v1',
-            kind='PersistentVolume',
-            metadata={'name': name},
-            spec=spec
-        )
+        V1PersistentVolume(api_version='v1', kind='PersistentVolume', metadata={'name': name}, spec=spec)
     )
diff --git a/liminal/logging/logging_setup.py b/liminal/logging/logging_setup.py
index 0765ef1..0619352 100644
--- a/liminal/logging/logging_setup.py
+++ b/liminal/logging/logging_setup.py
@@ -31,18 +31,13 @@ def logging_initialization():
     root_logger = logging.getLogger()
 
     log_formatter = logging.Formatter(
-        '[%(asctime)s] [%(filename)s:%(lineno)d] %(levelname)s - %(message)s',
-        '%m-%d %H:%M:%S'
+        '[%(asctime)s] [%(filename)s:%(lineno)d] %(levelname)s - %(message)s', '%m-%d %H:%M:%S'
     )
 
     logs_dir = os.path.join(environment.get_liminal_home(), LOGS_DIR)
     os.makedirs(logs_dir, exist_ok=True)
 
-    file_handler = RotatingFileHandler(
-        os.path.join(logs_dir, LOG_FILENAME),
-        maxBytes=MAX_FILE_SIZE,
-        backupCount=3
-    )
+    file_handler = RotatingFileHandler(os.path.join(logs_dir, LOG_FILENAME), maxBytes=MAX_FILE_SIZE, backupCount=3)
 
     root_logger.addHandler(file_handler)
     root_logger.setLevel(logging.INFO)
diff --git a/liminal/runners/airflow/__init__.py b/liminal/runners/airflow/__init__.py
index e7b2850..031328f 100644
--- a/liminal/runners/airflow/__init__.py
+++ b/liminal/runners/airflow/__init__.py
@@ -23,7 +23,6 @@ import pytz
 
 
 class DummyDagRun:
-
     def __init__(self) -> None:
         self.start_date = pytz.utc.localize(datetime.utcnow())
         self.conf = None
@@ -35,13 +34,14 @@ class DummyDagRun:
 
     def get_task_instance(self, _):
         return self
-      
+
+
 from datetime import datetime
 
 TASK_ID_SEPARATOR = '.'
 
-class DummyDag:
 
+class DummyDag:
     def __init__(self, dag_id, task_id):
         self.dag_id = dag_id
         self.task_id = task_id
@@ -57,7 +57,7 @@ class DummyDag:
             'task': self,
             'ti': self,
             'ts': datetime.now().timestamp(),
-            'dag_run': DummyDagRun()
+            'dag_run': DummyDagRun(),
         }
 
     def get_dagrun(self):
diff --git a/liminal/runners/airflow/dag/__init__.py b/liminal/runners/airflow/dag/__init__.py
index 6c5a8f5..8d028e2 100644
--- a/liminal/runners/airflow/dag/__init__.py
+++ b/liminal/runners/airflow/dag/__init__.py
@@ -19,7 +19,8 @@ import os
 from liminal.core import environment as env
 from liminal.runners.airflow.dag import liminal_register_dags
 
-BASE_PATH=os.path.join(env.get_airflow_home_dir(), env.DEFAULT_PIPELINES_SUBDIR)
+BASE_PATH = os.path.join(env.get_airflow_home_dir(), env.DEFAULT_PIPELINES_SUBDIR)
+
 
 def register_dags():
-    return liminal_register_dags.register_dags(BASE_PATH)
\ No newline at end of file
+    return liminal_register_dags.register_dags(BASE_PATH)
diff --git a/liminal/runners/airflow/dag/liminal_register_dags.py b/liminal/runners/airflow/dag/liminal_register_dags.py
index 77a729d..b5e921e 100644
--- a/liminal/runners/airflow/dag/liminal_register_dags.py
+++ b/liminal/runners/airflow/dag/liminal_register_dags.py
@@ -61,8 +61,7 @@ def register_dags(configs_path):
 
             executors = __initialize_executors(config)
 
-            default_executor = airflow.AirflowExecutor("default_executor", liminal_config=config,
-                                                       executor_config={})
+            default_executor = airflow.AirflowExecutor("default_executor", liminal_config=config, executor_config={})
 
             for pipeline in config['pipelines']:
                 default_args = __default_args(pipeline)
@@ -80,16 +79,18 @@ def register_dags(configs_path):
                         liminal_config=config,
                         pipeline_config=pipeline,
                         task_config=task,
-                        variables=config.get('variables', {})
+                        variables=config.get('variables', {}),
                     )
 
                     executor_id = task.get('executor')
                     if executor_id:
                         executor = executors[executor_id]
                     else:
-                        logging.info(f"Did not find `executor` in ${task['task']} config."
-                                     f" Using the default executor (${type(default_executor)})"
-                                     f" instead.")
+                        logging.info(
+                            f"Did not find `executor` in ${task['task']} config."
+                            f" Using the default executor (${type(default_executor)})"
+                            f" instead."
+                        )
                         executor = default_executor
 
                     parent = executor.apply_task_to_dag(task=task_instance)
@@ -109,9 +110,7 @@ def __initialize_executors(liminal_config):
     executors = {}
     for executor_config in liminal_config.get('executors', {}):
         executors[executor_config['executor']] = get_executor_class(executor_config['type'])(
-            executor_config['executor'],
-            liminal_config,
-            executor_config
+            executor_config['executor'], liminal_config, executor_config
         )
     return executors
 
@@ -141,7 +140,7 @@ def __initialize_dag(default_args, pipeline, owner):
         dagrun_timeout=timedelta(minutes=pipeline['timeout_minutes']),
         start_date=start_date,
         schedule_interval=schedule_interval,
-        catchup=False
+        catchup=False,
     )
 
     return dag
@@ -151,9 +150,7 @@ def __default_args(pipeline):
     default_args = {k: v for k, v in pipeline.items()}
     override_args = {
         'start_date': datetime.combine(pipeline['start_date'], datetime.min.time()),
-        __DEPENDS_ON_PAST: default_args[
-            __DEPENDS_ON_PAST
-        ] if __DEPENDS_ON_PAST in default_args else False,
+        __DEPENDS_ON_PAST: default_args[__DEPENDS_ON_PAST] if __DEPENDS_ON_PAST in default_args else False,
     }
     default_args.update(override_args)
     return default_args
@@ -170,8 +167,8 @@ tasks_by_liminal_name = class_util.find_subclasses_in_packages([impl_packages],
 logging.info(f'Finished loading task implementations: {tasks_by_liminal_name.keys()}')
 
 executors_by_liminal_name = class_util.find_subclasses_in_packages(
-    ['liminal.runners.airflow.executors'],
-    liminal_executor.Executor)
+    ['liminal.runners.airflow.executors'], liminal_executor.Executor
+)
 
 logging.info(f'Finished loading executor implementations: {executors_by_liminal_name.keys()}')
 
diff --git a/liminal/runners/airflow/executors/emr.py b/liminal/runners/airflow/executors/emr.py
index 9eb7ba3..dc5e7eb 100644
--- a/liminal/runners/airflow/executors/emr.py
+++ b/liminal/runners/airflow/executors/emr.py
@@ -50,11 +50,10 @@ class EMRExecutor(executor.Executor):
                 job_flow_id=self.job_flow_id,
                 job_flow_name=self.job_flow_name,
                 aws_conn_id=self.aws_conn_id,
-                steps=self.__generate_emr_step(task.task_id,
-                                               [str(x) for x in task.get_runnable_command()]),
+                steps=self.__generate_emr_step(task.task_id, [str(x) for x in task.get_runnable_command()]),
                 cluster_states=self.cluster_states,
             ),
-            task
+            task,
         )
 
         if task.parent:
@@ -63,13 +62,11 @@ class EMRExecutor(executor.Executor):
         emr_sensor_step = executor.add_variables_to_operator(
             EmrStepSensor(
                 task_id=f'{task.task_id}_watch_step',
-                job_flow_id="{{ task_instance.xcom_pull('" + add_step.task_id +
-                            "', key='job_flow_id') }}",
-                step_id="{{ task_instance.xcom_pull('" + add_step.task_id +
-                        "', key='return_value')[0] }}",
-                aws_conn_id=self.aws_conn_id
+                job_flow_id="{{ task_instance.xcom_pull('" + add_step.task_id + "', key='job_flow_id') }}",
+                step_id="{{ task_instance.xcom_pull('" + add_step.task_id + "', key='return_value')[0] }}",
+                aws_conn_id=self.aws_conn_id,
             ),
-            task
+            task,
         )
 
         add_step.set_downstream(emr_sensor_step)
@@ -81,9 +78,6 @@ class EMRExecutor(executor.Executor):
             {
                 'Name': task_id,
                 **self.executor_config.get('properties', {}),
-                'HadoopJarStep': {
-                    'Jar': 'command-runner.jar',
-                    'Args': args
-                }
+                'HadoopJarStep': {'Jar': 'command-runner.jar', 'Args': args},
             }
         ]
diff --git a/liminal/runners/airflow/executors/kubernetes.py b/liminal/runners/airflow/executors/kubernetes.py
index 7e69761..ca386cb 100644
--- a/liminal/runners/airflow/executors/kubernetes.py
+++ b/liminal/runners/airflow/executors/kubernetes.py
@@ -40,10 +40,8 @@ class KubernetesPodExecutor(executor.Executor):
 
     supported_task_types = [containerable.ContainerTask]
 
-    def __init__(self, task_id,
-                 liminal_config, executor_config):
-        super().__init__(task_id,
-                         liminal_config, executor_config)
+    def __init__(self, task_id, liminal_config, executor_config):
+        super().__init__(task_id, liminal_config, executor_config)
 
         self.task_name = self.executor_config['executor']
         self.volumes = self._volumes()
@@ -55,11 +53,7 @@ class KubernetesPodExecutor(executor.Executor):
         self._validate_task_type(task)
 
         pod_task = executor.add_variables_to_operator(
-            KubernetesPodOperator(
-                trigger_rule=task.trigger_rule,
-                **self.__kubernetes_kwargs(task)
-            ),
-            task
+            KubernetesPodOperator(trigger_rule=task.trigger_rule, **self.__kubernetes_kwargs(task)), task
         )
 
         if parent:
@@ -75,12 +69,7 @@ class KubernetesPodExecutor(executor.Executor):
             claim_name = volume_config.get('claim_name')
             if not claim_name and 'local' in volume_config:
                 claim_name = f'{name}-pvc'
-            volume = V1Volume(
-                name=name,
-                persistent_volume_claim={
-                    'claimName': claim_name
-                }
-            )
+            volume = V1Volume(name=name, persistent_volume_claim={'claimName': claim_name})
             volumes.append(volume)
         return volumes
 
@@ -105,13 +94,14 @@ class KubernetesPodExecutor(executor.Executor):
             'cluster_context': os.environ.get('AIRFLOW__KUBERNETES__CLUSTER_CONTEXT', None),
             'cmds': task.cmds,
             'volume_mounts': [
-                V1VolumeMount(name=mount['volume'],
-                              mount_path=mount['path'],
-                              sub_path=mount.get('sub_path'),
-                              read_only=mount.get('read_only', False))
-                for mount
-                in task.mounts
-            ]
+                V1VolumeMount(
+                    name=mount['volume'],
+                    mount_path=mount['path'],
+                    sub_path=mount.get('sub_path'),
+                    read_only=mount.get('read_only', False),
+                )
+                for mount in task.mounts
+            ],
         }
 
         config.pop('in_cluster', None)
@@ -127,9 +117,11 @@ class KubernetesPodExecutor(executor.Executor):
             kubernetes_kwargs['namespace'] = 'jenkins'
 
         if not task.dag:
-            kubernetes_kwargs.update({
-                'start_date': datetime.datetime(1970, 1, 1),
-            })
+            kubernetes_kwargs.update(
+                {
+                    'start_date': datetime.datetime(1970, 1, 1),
+                }
+            )
 
         return kubernetes_kwargs
 
@@ -140,20 +132,10 @@ class KubernetesPodExecutor(executor.Executor):
                 "requiredDuringSchedulingIgnoredDuringExecution": [
                     {
                         "labelSelector": {
-                            "matchExpressions": [
-                                {
-                                    "key": "liminal",
-                                    "operator": "In",
-                                    "values": [
-                                        "unittest"
-                                    ]
-                                }
-                            ]
+                            "matchExpressions": [{"key": "liminal", "operator": "In", "values": ["unittest"]}]
                         },
-                        "namespaces": [
-                            "jenkins"
-                        ],
-                        "topologyKey": "kubernetes.io/hostname"
+                        "namespaces": ["jenkins"],
+                        "topologyKey": "kubernetes.io/hostname",
                     }
                 ]
             }
diff --git a/liminal/runners/airflow/model/executor.py b/liminal/runners/airflow/model/executor.py
index 9e30aac..2af3a2d 100644
--- a/liminal/runners/airflow/model/executor.py
+++ b/liminal/runners/airflow/model/executor.py
@@ -20,8 +20,7 @@ from abc import ABC, abstractmethod
 
 from airflow.models import BaseOperator
 
-from liminal.runners.airflow.operators.operator_with_variable_resolving import \
-    OperatorWithVariableResolving
+from liminal.runners.airflow.operators.operator_with_variable_resolving import OperatorWithVariableResolving
 
 
 def add_variables_to_operator(operator, task) -> BaseOperator:
@@ -36,7 +35,7 @@ def add_variables_to_operator(operator, task) -> BaseOperator:
         task_config=task.task_config,
         variables=task.variables,
         liminal_task_instance=task,
-        operator=operator
+        operator=operator,
     )
 
 
@@ -44,6 +43,7 @@ class Executor(ABC):
     """
     Executor Task.
     """
+
     # list of task types supported by the executor
     supported_task_types = []
 
@@ -64,5 +64,6 @@ class Executor(ABC):
         pass
 
     def _validate_task_type(self, task):
-        assert any([isinstance(task, tYp) for tYp in self.supported_task_types]), \
-            f'supported task types: {self.supported_task_types}'
+        assert any(
+            [isinstance(task, tYp) for tYp in self.supported_task_types]
+        ), f'supported task types: {self.supported_task_types}'
diff --git a/liminal/runners/airflow/model/task.py b/liminal/runners/airflow/model/task.py
index e7a2493..26a62c3 100644
--- a/liminal/runners/airflow/model/task.py
+++ b/liminal/runners/airflow/model/task.py
@@ -32,8 +32,9 @@ class Task(ABC):
     Task.
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
         self.liminal_config = liminal_config
         self.dag = dag
         self.pipeline_config = pipeline_config
diff --git a/liminal/runners/airflow/operators/cloudformation.py b/liminal/runners/airflow/operators/cloudformation.py
index dfbc262..b846ef0 100644
--- a/liminal/runners/airflow/operators/cloudformation.py
+++ b/liminal/runners/airflow/operators/cloudformation.py
@@ -53,16 +53,13 @@ class BaseCloudFormationOperator(BaseOperator):
     :param aws_conn_id: aws connection to uses
     :type aws_conn_id: str
     """
+
     template_fields: List[str] = []
     template_ext = ()
     ui_color = '#1d472b'
     ui_fgcolor = '#FFF'
 
-    def __init__(
-            self,
-            params,
-            aws_conn_id='aws_default',
-            *args, **kwargs):
+    def __init__(self, params, aws_conn_id='aws_default', *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.params = params
         self.aws_conn_id = aws_conn_id
@@ -89,15 +86,12 @@ class CloudFormationCreateStackOperator(BaseCloudFormationOperator):
     :param aws_conn_id: aws connection to uses
     :type aws_conn_id: str
     """
+
     template_fields: List[str] = []
     template_ext = ()
     ui_color = '#6b9659'
 
-    def __init__(
-            self,
-            params,
-            aws_conn_id='aws_default',
-            *args, **kwargs):
+    def __init__(self, params, aws_conn_id='aws_default', *args, **kwargs):
         super().__init__(params=params, aws_conn_id=aws_conn_id, *args, **kwargs)
 
     def cloudformation_op(self, cloudformation):
@@ -114,16 +108,13 @@ class CloudFormationDeleteStackOperator(BaseCloudFormationOperator):
     :param aws_conn_id: aws connection to uses
     :type aws_conn_id: str
     """
+
     template_fields: List[str] = []
     template_ext = ()
     ui_color = '#1d472b'
     ui_fgcolor = '#FFF'
 
-    def __init__(
-            self,
-            params,
-            aws_conn_id='aws_default',
-            *args, **kwargs):
+    def __init__(self, params, aws_conn_id='aws_default', *args, **kwargs):
         super().__init__(params=params, aws_conn_id=aws_conn_id, *args, **kwargs)
 
     def cloudformation_op(self, cloudformation):
@@ -143,14 +134,16 @@ class BaseCloudFormationSensor(BaseSensorOperator):
     :type poke_interval: int
     """
 
-    def __init__(self,
-                 stack_name,
-                 complete_status,
-                 in_progress_status,
-                 aws_conn_id='aws_default',
-                 poke_interval=30,
-                 *args,
-                 **kwargs):
+    def __init__(
+        self,
+        stack_name,
+        complete_status,
+        in_progress_status,
+        aws_conn_id='aws_default',
+        poke_interval=30,
+        *args,
+        **kwargs,
+    ):
         super().__init__(poke_interval=poke_interval, *args, **kwargs)
         self.aws_conn_id = aws_conn_id
         self.stack_name = stack_name
@@ -216,19 +209,16 @@ class CloudFormationCreateStackSensor(BaseCloudFormationSensor):
     template_fields = ['stack_name']
     ui_color = '#C5CAE9'
 
-    def __init__(self,
-                 stack_name,
-                 aws_conn_id='aws_default',
-                 poke_interval=30,
-                 *args,
-                 **kwargs):
-        super().__init__(stack_name=stack_name,
-                         complete_status='CREATE_COMPLETE',
-                         in_progress_status='CREATE_IN_PROGRESS',
-                         aws_conn_id=aws_conn_id,
-                         poke_interval=poke_interval,
-                         *args,
-                         **kwargs)
+    def __init__(self, stack_name, aws_conn_id='aws_default', poke_interval=30, *args, **kwargs):
+        super().__init__(
+            stack_name=stack_name,
+            complete_status='CREATE_COMPLETE',
+            in_progress_status='CREATE_IN_PROGRESS',
+            aws_conn_id=aws_conn_id,
+            poke_interval=poke_interval,
+            *args,
+            **kwargs,
+        )
 
 
 class CloudFormationDeleteStackSensor(BaseCloudFormationSensor):
@@ -247,17 +237,16 @@ class CloudFormationDeleteStackSensor(BaseCloudFormationSensor):
     template_fields = ['stack_name']
     ui_color = '#C5CAE9'
 
-    def __init__(self,
-                 stack_name,
-                 aws_conn_id='aws_default',
-                 poke_interval=30,
-                 *args,
-                 **kwargs):
-        super().__init__(stack_name=stack_name,
-                         complete_status='DELETE_COMPLETE',
-                         in_progress_status='DELETE_IN_PROGRESS',
-                         aws_conn_id=aws_conn_id,
-                         poke_interval=poke_interval, *args, **kwargs)
+    def __init__(self, stack_name, aws_conn_id='aws_default', poke_interval=30, *args, **kwargs):
+        super().__init__(
+            stack_name=stack_name,
+            complete_status='DELETE_COMPLETE',
+            in_progress_status='DELETE_IN_PROGRESS',
+            aws_conn_id=aws_conn_id,
+            poke_interval=poke_interval,
+            *args,
+            **kwargs,
+        )
 
     def allow_non_existing_stack_status(self):
         return True
diff --git a/liminal/runners/airflow/operators/job_status_operator.py b/liminal/runners/airflow/operators/job_status_operator.py
index 6fc5ec5..60ecce9 100644
--- a/liminal/runners/airflow/operators/job_status_operator.py
+++ b/liminal/runners/airflow/operators/job_status_operator.py
@@ -33,13 +33,11 @@ class JobStatusOperator(BaseOperator):
     """
     Base operator for job status operators.
     """
+
     template_ext = ()
 
     @apply_defaults
-    def __init__(
-            self,
-            backends,
-            *args, **kwargs):
+    def __init__(self, backends, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.backends = backends
         self.cloudwatch = None
@@ -58,9 +56,7 @@ class JobStatusOperator(BaseOperator):
     def send_metric_to_cloudwatch(self, metric):
         self.get_cloudwatch().put_metric_data(metric)
 
-    report_functions = {
-        'cloudwatch': send_metric_to_cloudwatch
-    }
+    report_functions = {'cloudwatch': send_metric_to_cloudwatch}
 
     def get_cloudwatch(self):
         if not self.cloudwatch:
@@ -71,30 +67,19 @@ class JobStatusOperator(BaseOperator):
 class JobStartOperator(JobStatusOperator):
     ui_color = '#c5e5e8'
 
-    def __init__(
-            self,
-            namespace,
-            application_name,
-            backends,
-            *args, **kwargs):
+    def __init__(self, namespace, application_name, backends, *args, **kwargs):
         super().__init__(backends=backends, *args, **kwargs)
         self.namespace = namespace
         self.application_name = application_name
 
     def metrics(self, context):
-        return [Metric(self.namespace, 'JobStarted', 1,
-                       [Tag('ApplicationName', self.application_name)])]
+        return [Metric(self.namespace, 'JobStarted', 1, [Tag('ApplicationName', self.application_name)])]
 
 
 class JobEndOperator(JobStatusOperator):
     ui_color = '#6d8fad'
 
-    def __init__(
-            self,
-            namespace,
-            application_name,
-            backends,
-            *args, **kwargs):
+    def __init__(self, namespace, application_name, backends, *args, **kwargs):
         super().__init__(backends=backends, *args, **kwargs)
         self.namespace = namespace
         self.application_name = application_name
@@ -103,9 +88,11 @@ class JobEndOperator(JobStatusOperator):
     def execute(self, context):
         self.__calculate_job_result(context)
         super().execute(context)
-        
+
     def metrics(self, context):
-        duration = round((pytz.utc.localize(datetime.utcnow()) - context['ti'].get_dagrun().start_date).total_seconds())
+        duration = round(
+            (pytz.utc.localize(datetime.utcnow()) - context['ti'].get_dagrun().start_date).total_seconds()
+        )
 
         self.log.info('Elapsed time: %s' % duration)
 
@@ -113,7 +100,7 @@ class JobEndOperator(JobStatusOperator):
 
         return [
             Metric(self.namespace, 'JobResult', self.__job_result, [Tag('ApplicationName', self.application_name)]),
-            Metric(self.namespace, 'JobDuration', duration, [Tag('ApplicationName', self.application_name)])
+            Metric(self.namespace, 'JobDuration', duration, [Tag('ApplicationName', self.application_name)]),
         ]
 
     def __log_and_get_state(self, task_instance):
@@ -122,13 +109,15 @@ class JobEndOperator(JobStatusOperator):
         self.log.info(f'Task {task_instance.task_id} finished with state = {state}')
 
         return state
-    
+
     def __calculate_job_result(self, context):
         self.log.info('scanning task instances states.. ')
         task_instances = context['dag_run'].get_task_instances()
-        task_states = [self.__log_and_get_state(task_instance)
-                       for task_instance in task_instances
-                       if task_instance.task_id != context['task_instance'].task_id]
+        task_states = [
+            self.__log_and_get_state(task_instance)
+            for task_instance in task_instances
+            if task_instance.task_id != context['task_instance'].task_id
+        ]
 
         self.__job_result = 0
         if all((state == State.SUCCESS or state == State.SKIPPED) for state in task_states):
@@ -176,9 +165,9 @@ class CloudWatchHook(AwsHook):
                     'Dimensions': dimensions,
                     'Timestamp': datetime.utcnow(),
                     'Value': value,
-                    'Unit': 'None'
+                    'Unit': 'None',
                 }
-            ]
+            ],
         )
 
         self.log.info(f'Published metric: {metric.name} with value: {value}')
@@ -197,12 +186,7 @@ class Metric:
     :type tags: List[str]
     """
 
-    def __init__(
-            self,
-            namespace,
-            name,
-            value,
-            tags):
+    def __init__(self, namespace, name, value, tags):
         self.namespace = namespace
         self.name = name
         self.value = value
@@ -210,9 +194,6 @@ class Metric:
 
 
 class Tag:
-    def __init__(
-            self,
-            name,
-            value):
+    def __init__(self, name, value):
         self.name = name
         self.value = value
diff --git a/liminal/runners/airflow/operators/operator_with_variable_resolving.py b/liminal/runners/airflow/operators/operator_with_variable_resolving.py
index d363ba1..9f00f8e 100644
--- a/liminal/runners/airflow/operators/operator_with_variable_resolving.py
+++ b/liminal/runners/airflow/operators/operator_with_variable_resolving.py
@@ -39,35 +39,23 @@ class OperatorWithVariableResolving(BaseOperator):
     Operator delegator that handles liminal variable substitution at run time
     """
 
-    def __init__(self,
-                 dag,
-                 task_config: dict,
-                 variables: dict = None,
-                 liminal_task_instance=None,
-                 **kwargs):
+    def __init__(self, dag, task_config: dict, variables: dict = None, liminal_task_instance=None, **kwargs):
         self.operator_delegate: BaseOperator = kwargs.pop('operator')
-        self.liminal_task_instance = \
-            liminal_task_instance.serialize() if liminal_task_instance else None
+        self.liminal_task_instance = liminal_task_instance.serialize() if liminal_task_instance else None
         if variables:
             self.variables = variables.copy()
         else:
             self.variables = {}
         self.task_config = task_config
-        super().__init__(
-            task_id=self.operator_delegate.task_id,
-            dag=dag
-        )
+        super().__init__(task_id=self.operator_delegate.task_id, dag=dag)
         self._LOG = logging.getLogger(self.__class__.__name__)
 
     def execute(self, context):
         attributes = self._get_operator_delegate_attributes()
         self._LOG.info(f'task_config: {self.task_config}')
         self._LOG.info(f'variables: {self.variables}')
-        self.operator_delegate.template_fields = set(list(self.operator_delegate.template_fields) +
-                                                     attributes)
-        self.operator_delegate.render_template_fields(context,
-                                                      LiminalEnvironment(self.variables,
-                                                                         self.task_config))
+        self.operator_delegate.template_fields = set(list(self.operator_delegate.template_fields) + attributes)
+        self.operator_delegate.render_template_fields(context, LiminalEnvironment(self.variables, self.task_config))
         self.operator_delegate.render_template_fields(context)
 
         if 'ti' in context:
@@ -80,8 +68,10 @@ class OperatorWithVariableResolving(BaseOperator):
 
     def _get_operator_delegate_attributes(self):
         return [
-            attr for attr in dir(self.operator_delegate) if
-            attr not in _BASE_OPERATOR_ATTRIBUTES and attr not in dir(BaseOperator)
+            attr
+            for attr in dir(self.operator_delegate)
+            if attr not in _BASE_OPERATOR_ATTRIBUTES
+            and attr not in dir(BaseOperator)
             and not attr.startswith('_')
             and attr not in ('args', 'kwargs', 'lineage_data', 'subdag', 'template_fields')
         ]
@@ -92,16 +82,19 @@ class OperatorWithVariableResolving(BaseOperator):
     def on_kill(self) -> None:
         self.operator_delegate.on_kill()
 
-    def render_template_fields(self, context: Dict,
-                               jinja_env: Optional[jinja2.Environment] = None) -> None:
+    def render_template_fields(self, context: Dict, jinja_env: Optional[jinja2.Environment] = None) -> None:
         pass
 
-    def render_template(self, content: Any, context: Dict,
-                        jinja_env: Optional[jinja2.Environment] = None,
-                        seen_oids: Optional[Set] = None) -> Any:
-        value = self.operator_delegate.render_template(content, context,
-                                                       LiminalEnvironment(self.variables,
-                                                                          self.task_config))
+    def render_template(
+        self,
+        content: Any,
+        context: Dict,
+        jinja_env: Optional[jinja2.Environment] = None,
+        seen_oids: Optional[Set] = None,
+    ) -> Any:
+        value = self.operator_delegate.render_template(
+            content, context, LiminalEnvironment(self.variables, self.task_config)
+        )
         return self.operator_delegate.render_template(value, context, jinja_env, seen_oids)
 
     def get_template_env(self) -> jinja2.Environment:
@@ -113,16 +106,25 @@ class OperatorWithVariableResolving(BaseOperator):
     def resolve_template_files(self) -> None:
         self.operator_delegate.resolve_template_files()
 
-    def clear(self, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None,
-              upstream: bool = False, downstream: bool = False, session: Session = None):
+    def clear(
+        self,
+        start_date: Optional[datetime] = None,
+        end_date: Optional[datetime] = None,
+        upstream: bool = False,
+        downstream: bool = False,
+        session: Session = None,
+    ):
         return self.operator_delegate.clear(start_date, end_date, upstream, downstream, session)
 
-    def run(self, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None,
-            ignore_first_depends_on_past: bool = True, ignore_ti_state: bool = False,
-            mark_success: bool = False) -> None:
-        self.operator_delegate.run(start_date, end_date, ignore_first_depends_on_past,
-                                   ignore_ti_state,
-                                   mark_success)
+    def run(
+        self,
+        start_date: Optional[datetime] = None,
+        end_date: Optional[datetime] = None,
+        ignore_first_depends_on_past: bool = True,
+        ignore_ti_state: bool = False,
+        mark_success: bool = False,
+    ) -> None:
+        self.operator_delegate.run(start_date, end_date, ignore_first_depends_on_past, ignore_ti_state, mark_success)
 
 
 class LiminalEnvironment(Environment):
@@ -158,28 +160,16 @@ class LiminalEnvironment(Environment):
             prefix = self.__render(token[1], dag_run_conf, unresolved_tags)
             suffix = self.__render(token[3], dag_run_conf, unresolved_tags)
             if dag_run_conf and tag_name in dag_run_conf:
-                return self.__render(prefix + str(dag_run_conf[tag_name]) + suffix,
-                                     dag_run_conf,
-                                     unresolved_tags)
+                return self.__render(prefix + str(dag_run_conf[tag_name]) + suffix, dag_run_conf, unresolved_tags)
             elif tag_name in self.variables:
-                return self.__render(prefix + str(self.variables[tag_name]) + suffix,
-                                     dag_run_conf,
-                                     unresolved_tags)
+                return self.__render(prefix + str(self.variables[tag_name]) + suffix, dag_run_conf, unresolved_tags)
             else:
                 backend_value = standalone_variable_backend.get_variable(tag_name, None)
                 if backend_value:
-                    return self.__render(
-                        prefix + backend_value + suffix,
-                        dag_run_conf,
-                        unresolved_tags
-                    )
+                    return self.__render(prefix + backend_value + suffix, dag_run_conf, unresolved_tags)
                 else:
                     unresolved_tags.add(tag_name)
-                    return self.__render(
-                        prefix + '{{' + token[2] + '}}' + suffix,
-                        dag_run_conf,
-                        unresolved_tags
-                    )
+                    return self.__render(prefix + '{{' + token[2] + '}}' + suffix, dag_run_conf, unresolved_tags)
         else:
             return val
 
@@ -197,5 +187,5 @@ def add_variables_to_operator(operator, task) -> BaseOperator:
         task_config=task.task_config,
         variables=task.variables,
         liminal_task_instance=task,
-        operator=operator
+        operator=operator,
     )
diff --git a/liminal/runners/airflow/tasks/containerable.py b/liminal/runners/airflow/tasks/containerable.py
index 08e1139..46028e3 100644
--- a/liminal/runners/airflow/tasks/containerable.py
+++ b/liminal/runners/airflow/tasks/containerable.py
@@ -37,10 +37,10 @@ class ContainerTask(task.Task, ABC):
     K8S Containerable task
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
-        super().__init__(task_id, dag, parent, trigger_rule, liminal_config,
-                         pipeline_config, task_config, variables)
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
+        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables)
         env = standalone_variable_backend.get_variable(ENV, DEFAULT)
         self.env_vars = self.__env_vars(env)
         self.image = self.task_config['image']
@@ -50,9 +50,7 @@ class ContainerTask(task.Task, ABC):
     def _kubernetes_cmds_and_arguments(self):
         cmds = ['/bin/sh', '-c']
 
-        arguments = [
-            self.task_config['cmd']
-        ]
+        arguments = [self.task_config['cmd']]
 
         return cmds, arguments
 
@@ -77,8 +75,7 @@ class ContainerTask(task.Task, ABC):
         env_vars = dict(self.task_config['env_vars']) if 'env_vars' in self.task_config else {}
         env_vars.update(self.__get_local_env_params_from_env_file())
         airflow_configuration_variable = get_variable(
-            f'''{self.pipeline_config['pipeline']}_dag_configuration''',
-            default_val=None
+            f'''{self.pipeline_config['pipeline']}_dag_configuration''', default_val=None
         )
 
         if airflow_configuration_variable:
diff --git a/liminal/runners/airflow/tasks/create_cloudformation_stack.py b/liminal/runners/airflow/tasks/create_cloudformation_stack.py
index 2ac1605..cbc457d 100644
--- a/liminal/runners/airflow/tasks/create_cloudformation_stack.py
+++ b/liminal/runners/airflow/tasks/create_cloudformation_stack.py
@@ -20,10 +20,12 @@ from airflow.operators.dummy import DummyOperator
 from airflow.operators.python import BranchPythonOperator
 from flatdict import FlatDict
 
-from liminal.runners.airflow.operators.cloudformation import CloudFormationCreateStackOperator, \
-    CloudFormationCreateStackSensor, CloudFormationHook
-from liminal.runners.airflow.operators.operator_with_variable_resolving import \
-    OperatorWithVariableResolving
+from liminal.runners.airflow.operators.cloudformation import (
+    CloudFormationCreateStackOperator,
+    CloudFormationCreateStackSensor,
+    CloudFormationHook,
+)
+from liminal.runners.airflow.operators.operator_with_variable_resolving import OperatorWithVariableResolving
 from liminal.runners.airflow.tasks import airflow
 
 
@@ -32,10 +34,10 @@ class CreateCloudFormationStackTask(airflow.AirflowTask):
     Creates cloud_formation stack.
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
-        super().__init__(task_id, dag, parent, trigger_rule, liminal_config,
-                         pipeline_config, task_config, variables)
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
+        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables)
         self.stack_name = task_config['stack_name']
 
     def apply_task_to_dag(self):
@@ -50,10 +52,7 @@ class CreateCloudFormationStackTask(airflow.AirflowTask):
 
         create_cloudformation_stack_task = self._add_variables_to_operator(
             CloudFormationCreateStackOperator(
-                task_id=f'create-cloudformation-{self.task_id}',
-                params={
-                    **self.__reformatted_params()
-                }
+                task_id=f'create-cloudformation-{self.task_id}', params={**self.__reformatted_params()}
             )
         )
 
@@ -65,9 +64,7 @@ class CreateCloudFormationStackTask(airflow.AirflowTask):
         )
 
         stack_creation_end_task = DummyOperator(
-            task_id=f'creation-end-{self.task_id}',
-            trigger_rule='all_done',
-            dag=self.dag
+            task_id=f'creation-end-{self.task_id}', trigger_rule='all_done', dag=self.dag
         )
 
         if self.parent:
@@ -83,8 +80,7 @@ class CreateCloudFormationStackTask(airflow.AirflowTask):
     def __cloudformation_stack_running_branch(self, stack_name):
         cloudformation = CloudFormationHook().get_conn()
         try:
-            stack_status = cloudformation.describe_stacks(StackName=stack_name)['Stacks'][0][
-                'StackStatus']
+            stack_status = cloudformation.describe_stacks(StackName=stack_name)['Stacks'][0]['StackStatus']
             if stack_status in ['CREATE_COMPLETE', 'DELETE_FAILED']:
                 print(f'Stack {stack_name} is running')
                 return f'creation-end-{self.task_id}'
@@ -104,6 +100,8 @@ class CreateCloudFormationStackTask(airflow.AirflowTask):
         return {
             'StackName': self.stack_name,
             **self.task_config['properties'],
-            'Parameters': [{'ParameterKey': x, 'ParameterValue': str(y)} for (x, y) in
-                           FlatDict(self.task_config['properties']['Parameters']).items()]
+            'Parameters': [
+                {'ParameterKey': x, 'ParameterValue': str(y)}
+                for (x, y) in FlatDict(self.task_config['properties']['Parameters']).items()
+            ],
         }
diff --git a/liminal/runners/airflow/tasks/delete_cloudformation_stack.py b/liminal/runners/airflow/tasks/delete_cloudformation_stack.py
index 756f645..57f1979 100644
--- a/liminal/runners/airflow/tasks/delete_cloudformation_stack.py
+++ b/liminal/runners/airflow/tasks/delete_cloudformation_stack.py
@@ -20,8 +20,10 @@ from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.python_operator import BranchPythonOperator
 from airflow.utils.trigger_rule import TriggerRule
 
-from liminal.runners.airflow.operators.cloudformation import CloudFormationDeleteStackOperator, \
-    CloudFormationDeleteStackSensor
+from liminal.runners.airflow.operators.cloudformation import (
+    CloudFormationDeleteStackOperator,
+    CloudFormationDeleteStackSensor,
+)
 from liminal.runners.airflow.tasks import airflow
 
 
@@ -30,10 +32,10 @@ class DeleteCloudFormationStackTask(airflow.AirflowTask):
     Deletes cloud_formation stack.
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
-        super().__init__(task_id, dag, parent, trigger_rule, liminal_config,
-                         pipeline_config, task_config, variables)
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
+        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables)
         self.stack_name = task_config['stack_name']
 
     def apply_task_to_dag(self):
@@ -42,7 +44,7 @@ class DeleteCloudFormationStackTask(airflow.AirflowTask):
             python_callable=self.__queued_dag_runs_exists,
             provide_context=True,
             trigger_rule=TriggerRule.ALL_DONE,
-            dag=self.dag
+            dag=self.dag,
         )
 
         delete_stack_task = self._add_variables_to_operator(
@@ -59,10 +61,7 @@ class DeleteCloudFormationStackTask(airflow.AirflowTask):
             )
         )
 
-        stack_delete_end_task = DummyOperator(
-            task_id=f'delete-end-{self.task_id}',
-            dag=self.dag
-        )
+        stack_delete_end_task = DummyOperator(task_id=f'delete-end-{self.task_id}', dag=self.dag)
 
         if self.parent:
             self.parent.set_downstream(check_dags_queued_task)
diff --git a/liminal/runners/airflow/tasks/job_end.py b/liminal/runners/airflow/tasks/job_end.py
index 4f8c295..9b65655 100644
--- a/liminal/runners/airflow/tasks/job_end.py
+++ b/liminal/runners/airflow/tasks/job_end.py
@@ -22,13 +22,13 @@ from liminal.runners.airflow.tasks import airflow
 
 class JobEndTask(airflow.AirflowTask):
     """
-      Job end task. Reports job end metrics.
+    Job end task. Reports job end metrics.
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
-        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                         task_config, variables)
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
+        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables)
         metrics = self.liminal_config.get('metrics', {})
         self.metrics_namespace = metrics.get('namespace', '')
         self.metrics_backends = metrics.get('backends', [])
@@ -40,7 +40,7 @@ class JobEndTask(airflow.AirflowTask):
             application_name=self.pipeline_config['pipeline'],
             backends=self.metrics_backends,
             dag=self.dag,
-            trigger_rule=self.trigger_rule
+            trigger_rule=self.trigger_rule,
         )
 
         if self.parent:
diff --git a/liminal/runners/airflow/tasks/job_start.py b/liminal/runners/airflow/tasks/job_start.py
index 78b69a2..a2d199c 100644
--- a/liminal/runners/airflow/tasks/job_start.py
+++ b/liminal/runners/airflow/tasks/job_start.py
@@ -25,10 +25,10 @@ class JobStartTask(airflow.AirflowTask):
     Job start task. Reports job start metrics.
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
-        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                         task_config, variables)
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
+        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables)
         metrics = self.liminal_config.get('metrics', {})
         self.metrics_namespace = metrics.get('namespace', '')
         self.metrics_backends = metrics.get('backends', [])
@@ -40,7 +40,7 @@ class JobStartTask(airflow.AirflowTask):
             application_name=self.pipeline_config['pipeline'],
             backends=self.metrics_backends,
             dag=self.dag,
-            trigger_rule=self.trigger_rule
+            trigger_rule=self.trigger_rule,
         )
 
         if self.parent:
diff --git a/liminal/runners/airflow/tasks/python.py b/liminal/runners/airflow/tasks/python.py
index 81163dd..f7c734d 100644
--- a/liminal/runners/airflow/tasks/python.py
+++ b/liminal/runners/airflow/tasks/python.py
@@ -26,10 +26,7 @@ class PythonTask(containerable.ContainerTask):
 
     def _kubernetes_cmds_and_arguments(self):
         cmds = ['/bin/bash', '-c']
-        arguments = [
-
-            self.__cmd()
-        ]
+        arguments = [self.__cmd()]
 
         return cmds, arguments
 
diff --git a/liminal/runners/airflow/tasks/spark.py b/liminal/runners/airflow/tasks/spark.py
index 8938011..d490bfd 100644
--- a/liminal/runners/airflow/tasks/spark.py
+++ b/liminal/runners/airflow/tasks/spark.py
@@ -28,13 +28,13 @@ class SparkTask(hadoop.HadoopTask, containerable.ContainerTask):
     Executes a Spark application.
     """
 
-    def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
-                 task_config, variables=None):
+    def __init__(
+        self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables=None
+    ):
         task_config['image'] = task_config.get('image', '')
         task_config['cmd'] = task_config.get('cmd', [])
         task_config['env_vars'] = {'SPARK_LOCAL_HOSTNAME': 'localhost'}
-        super().__init__(task_id, dag, parent, trigger_rule, liminal_config,
-                         pipeline_config, task_config, variables)
+        super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config, variables)
 
     def get_runnable_command(self):
         """
@@ -59,13 +59,12 @@ class SparkTask(hadoop.HadoopTask, containerable.ContainerTask):
 
         spark_arguments = {
             'master': self.task_config.get('master', None),
-            'class': self.task_config.get('class', None)
+            'class': self.task_config.get('class', None),
         }
 
         source_code = self.task_config.get("application_source")
 
-        for conf_arg in ['{}={}'.format(k, v) for (k, v) in
-                         FlatDict(self.task_config.get('conf', {})).items()]:
+        for conf_arg in ['{}={}'.format(k, v) for (k, v) in FlatDict(self.task_config.get('conf', {})).items()]:
             flat_conf_args.append('--conf')
             flat_conf_args.append(conf_arg)
 
@@ -83,8 +82,9 @@ class SparkTask(hadoop.HadoopTask, containerable.ContainerTask):
     def __parse_spark_arguments(self, spark_arguments):
         spark_arguments = {x[0]: x[1] for x in spark_arguments.items() if x[1]}
 
-        return self.__interleaving([f'--{k}' for k in spark_arguments.keys() if spark_arguments[k]],
-                                   spark_arguments.values())
+        return self.__interleaving(
+            [f'--{k}' for k in spark_arguments.keys() if spark_arguments[k]], spark_arguments.values()
+        )
 
     @staticmethod
     def __interleaving(keys, values):
diff --git a/liminal/runners/airflow/dag/__init__.py b/pyproject.toml
similarity index 71%
copy from liminal/runners/airflow/dag/__init__.py
copy to pyproject.toml
index 6c5a8f5..618c319 100644
--- a/liminal/runners/airflow/dag/__init__.py
+++ b/pyproject.toml
@@ -15,11 +15,23 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import os
-from liminal.core import environment as env
-from liminal.runners.airflow.dag import liminal_register_dags
 
-BASE_PATH=os.path.join(env.get_airflow_home_dir(), env.DEFAULT_PIPELINES_SUBDIR)
-
-def register_dags():
-    return liminal_register_dags.register_dags(BASE_PATH)
\ No newline at end of file
+[tool.black]
+line-length = 119  # TODO: Decide if 119 or 110?
+target-version = ['py36', 'py37', 'py38', 'py39']
+skip-string-normalization = true
+include = '\.pyi?$'
+exclude = '''
+(
+  /(
+      \.eggs
+    | \.git
+    | \.mypy_cache
+    | \.tox
+    | _build
+    | build
+    | dist
+  )/
+  | src/buildstream/_protos
+)
+'''
diff --git a/requirements.txt b/requirements.txt
index 415ee41..e24b824 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -45,3 +45,4 @@ apache-airflow-providers-cncf-kubernetes==1.0.2
 #apache-airflow-providers-google==4.0.0
 #apache-airflow[google,amazon,apache.spark]==2.0.0
 cfn-lint==0.53.0
+pre-commit==2.16.0
diff --git a/scripts/webserver_config.py b/scripts/webserver_config.py
index d7e953d..015ce9a 100644
--- a/scripts/webserver_config.py
+++ b/scripts/webserver_config.py
@@ -125,4 +125,4 @@ AUTH_ROLE_PUBLIC = 'Admin'
 # APP_THEME = "spacelab.css"
 # APP_THEME = "superhero.css"
 # APP_THEME = "united.css"
-# APP_THEME = "yeti.css"
\ No newline at end of file
+# APP_THEME = "yeti.css"
diff --git a/setup.py b/setup.py
index eae8d43..11b0105 100644
--- a/setup.py
+++ b/setup.py
@@ -50,7 +50,12 @@ setuptools.setup(
     scripts=['scripts/liminal'],
     include_package_data=True,
     package_data={
-        'liminal': ['../DISCLAIMER', '../LICENSE', '../NOTICE', '../README.md',
-                    '../liminal/core/config/defaults/base/liminal.yml'],
-    }
+        'liminal': [
+            '../DISCLAIMER',
+            '../LICENSE',
+            '../NOTICE',
+            '../README.md',
+            '../liminal/core/config/defaults/base/liminal.yml',
+        ],
+    },
 )
diff --git a/tests/liminal/core/config/defaults/test_apply_variables_substitution.py b/tests/liminal/core/config/defaults/test_apply_variables_substitution.py
index 02a816e..34e3547 100644
--- a/tests/liminal/core/config/defaults/test_apply_variables_substitution.py
+++ b/tests/liminal/core/config/defaults/test_apply_variables_substitution.py
@@ -23,47 +23,25 @@ from liminal.core.config.defaults import default_configs
 
 class TestApplyVariablesSubstitution(TestCase):
     def test_apply(self):
-        subliminal = {'variables': {
-            'one': 'one_value',
-            'two': 'two_value'
-        }}
+        subliminal = {'variables': {'one': 'one_value', 'two': 'two_value'}}
 
-        superliminal = {'variables': {
-            'three': 'three_value',
-            'two': 'two_super_value'
-        }}
+        superliminal = {'variables': {'three': 'three_value', 'two': 'two_super_value'}}
 
-        expected = {'variables': {
-            'one': 'one_value',
-            'two': 'two_value',
-            'three': 'three_value'
-        }}
+        expected = {'variables': {'one': 'one_value', 'two': 'two_value', 'three': 'three_value'}}
         self.assertEqual(expected, default_configs.apply_variable_substitution(subliminal, superliminal))
 
     def test_apply_superliminal_is_missing_variables(self):
-        subliminal = {'variables': {
-            'one': 'one_value',
-            'two': 'two_value'
-        }}
+        subliminal = {'variables': {'one': 'one_value', 'two': 'two_value'}}
 
         superliminal = {}
 
-        expected = {'variables': {
-            'one': 'one_value',
-            'two': 'two_value'
-        }}
+        expected = {'variables': {'one': 'one_value', 'two': 'two_value'}}
         self.assertEqual(expected, default_configs.apply_variable_substitution(subliminal, superliminal))
 
     def test_apply_subliminal_is_missing_variables(self):
         subliminal = {}
 
-        superliminal = {'variables': {
-            'one': 'one_value',
-            'two': 'two_value'
-        }}
+        superliminal = {'variables': {'one': 'one_value', 'two': 'two_value'}}
 
-        expected = {'variables': {
-            'one': 'one_value',
-            'two': 'two_value'
-        }}
+        expected = {'variables': {'one': 'one_value', 'two': 'two_value'}}
         self.assertEqual(expected, default_configs.apply_variable_substitution(subliminal, superliminal))
diff --git a/tests/liminal/core/config/defaults/test_defaults_pipeline_config.py b/tests/liminal/core/config/defaults/test_defaults_pipeline_config.py
index f3f5381..2d2f4bb 100644
--- a/tests/liminal/core/config/defaults/test_defaults_pipeline_config.py
+++ b/tests/liminal/core/config/defaults/test_defaults_pipeline_config.py
@@ -23,16 +23,7 @@ from liminal.core.config.defaults import default_configs
 
 class TestDefaultsPipelineConfig(TestCase):
     def test_apply(self):
-        pipeline = {
-            "name": "mypipe",
-            "param": "constant",
-            "tasks": [
-                {
-                    "task": "middle_task",
-                    "type": "python"
-                }
-            ]
-        }
+        pipeline = {"name": "mypipe", "param": "constant", "tasks": [{"task": "middle_task", "type": "python"}]}
 
         subliminal = {
             "name": "my_subliminal_test",
@@ -40,46 +31,24 @@ class TestDefaultsPipelineConfig(TestCase):
                 {
                     "name": "mypipe",
                     "param": "constant",
-                    "tasks": [
-                        {
-                            "task": "middle_task",
-                            "type": "python",
-                            "env_vars": {
-                                "env1": "env1"
-                            }
-                        }
-                    ]
+                    "tasks": [{"task": "middle_task", "type": "python", "env_vars": {"env1": "env1"}}],
                 }
             ],
-            "pipeline_defaults": {
-                "param1": "param1_value"
-            }
+            "pipeline_defaults": {"param1": "param1_value"},
         }
         superliminal = {
             "pipeline_defaults": {
                 "param2": "param2super_value",
                 "param3": "param3super_value",
-                "before_tasks": [
-                    {
-                        "task": "first_task",
-                        "type": "python"
-                    }],
-                "after_tasks": [
-                    {
-                        "task": "end_task",
-                        "type": "python"
-                    }
-                ]
+                "before_tasks": [{"task": "first_task", "type": "python"}],
+                "after_tasks": [{"task": "end_task", "type": "python"}],
             },
             "task_defaults": {
                 "python": {
                     "default_task_super": "default_task_super_value",
-                    "env_vars": {
-                        "env1": "env1super",
-                        "env2": "env2super"
-                    }
+                    "env_vars": {"env1": "env1super", "env2": "env2super"},
                 }
-            }
+            },
         }
         expected = {
             "param": "constant",
@@ -88,35 +57,25 @@ class TestDefaultsPipelineConfig(TestCase):
             "tasks": [
                 {
                     "type": "python",
-                    "env_vars": {
-                        "env1": "env1super",
-                        "env2": "env2super"
-                    },
+                    "env_vars": {"env1": "env1super", "env2": "env2super"},
                     "task": "first_task",
-                    "default_task_super": "default_task_super_value"
+                    "default_task_super": "default_task_super_value",
                 },
                 {
                     "type": "python",
-                    "env_vars": {
-                        "env1": "env1super",
-                        "env2": "env2super"
-                    },
+                    "env_vars": {"env1": "env1super", "env2": "env2super"},
                     "task": "middle_task",
-                    "default_task_super": "default_task_super_value"
+                    "default_task_super": "default_task_super_value",
                 },
                 {
                     "type": "python",
-                    "env_vars": {
-                        "env1": "env1super",
-                        "env2": "env2super"
-                    },
+                    "env_vars": {"env1": "env1super", "env2": "env2super"},
                     "task": "end_task",
-                    "default_task_super": "default_task_super_value"
-                }
+                    "default_task_super": "default_task_super_value",
+                },
             ],
             "name": "mypipe",
-            "param1": "param1_value"
+            "param1": "param1_value",
         }
 
-        self.assertEqual(expected, default_configs.apply_pipeline_defaults(subliminal, superliminal,
-                                                                           pipeline))
+        self.assertEqual(expected, default_configs.apply_pipeline_defaults(subliminal, superliminal, pipeline))
diff --git a/tests/liminal/core/config/defaults/test_defaults_service_config.py b/tests/liminal/core/config/defaults/test_defaults_service_config.py
index 13062df..c7523e5 100644
--- a/tests/liminal/core/config/defaults/test_defaults_service_config.py
+++ b/tests/liminal/core/config/defaults/test_defaults_service_config.py
@@ -25,34 +25,28 @@ class TestDefaultsServiceConfig(TestCase):
     def test_apply(self):
         subliminal = {
             "services": [
-                {
-                    "name": "my_python_server",
-                    "type": "python_server",
-                    "image": "default_image"
-                },
-                {
-                    "name": "my_python_server_for_stg",
-                    "type": "python_server",
-                    "image": "test_image"
-                }
-            ]}
-
-        superliminal = {
-            "service_defaults": {
-                "param": "param1",
-                "param2": "param2"
-            }
+                {"name": "my_python_server", "type": "python_server", "image": "default_image"},
+                {"name": "my_python_server_for_stg", "type": "python_server", "image": "test_image"},
+            ]
         }
 
-        expected = [{'image': 'default_image',
-                     'name': 'my_python_server',
-                     'param': 'param1',
-                     'param2': 'param2',
-                     'type': 'python_server'},
-                    {'image': 'test_image',
-                     'name': 'my_python_server_for_stg',
-                     'param': 'param1',
-                     'param2': 'param2',
-                     'type': 'python_server'}]
+        superliminal = {"service_defaults": {"param": "param1", "param2": "param2"}}
+
+        expected = [
+            {
+                'image': 'default_image',
+                'name': 'my_python_server',
+                'param': 'param1',
+                'param2': 'param2',
+                'type': 'python_server',
+            },
+            {
+                'image': 'test_image',
+                'name': 'my_python_server_for_stg',
+                'param': 'param1',
+                'param2': 'param2',
+                'type': 'python_server',
+            },
+        ]
 
         self.assertEqual(expected, default_configs.apply_service_defaults(subliminal, superliminal))
diff --git a/tests/liminal/core/config/defaults/test_defaults_tasks_config.py b/tests/liminal/core/config/defaults/test_defaults_tasks_config.py
index caa1c45..8ced00c 100644
--- a/tests/liminal/core/config/defaults/test_defaults_tasks_config.py
+++ b/tests/liminal/core/config/defaults/test_defaults_tasks_config.py
@@ -26,150 +26,103 @@ class TestDefaultsTaskConfig(TestCase):
         pipeline = {
             'pipeline': 'mypipe',
             "tasks": [
-                {
-                    "task": "middle",
-                    "type": "spark",
-                    "task_param": "task_middle_param"
-                },
-                {
-                    "task": "end",
-                    "type": "python",
-                    "task_param": "task_end_param"
-                }
-            ]
+                {"task": "middle", "type": "spark", "task_param": "task_middle_param"},
+                {"task": "end", "type": "python", "task_param": "task_end_param"},
+            ],
         }
 
         subliminal = {'pipelines': [pipeline]}
 
         superliminal = {
             "task_defaults": {
-                "python": {
-                    "env_vars": {
-                        "env2": "env2value"
-                    }
-                },
-                "spark": {
-                    "task_param": "task_spark_param",
-                    "executor": "emr"
-                }
+                "python": {"env_vars": {"env2": "env2value"}},
+                "spark": {"task_param": "task_spark_param", "executor": "emr"},
             }
         }
 
-        expected = {'pipeline': 'mypipe',
-                    'tasks': [{'env_vars': {'env1': 'env1value', 'env2': 'env2value'},
-                               'task': 'start',
-                               'type': 'python'},
-                              {'executor': 'emr',
-                               'task': 'middle',
-                               'task_param': 'task_middle_param',
-                               'type': 'spark'},
-                              {'env_vars': {'env2': 'env2value'},
-                               'task': 'end',
-                               'task_param': 'task_end_param',
-                               'type': 'python'}]}
-        self.assertEqual(expected, default_configs.apply_task_defaults(subliminal,
-                                                                       superliminal,
-                                                                       pipeline=pipeline,
-                                                                       superliminal_before_tasks=[{
-                                                                           "task": "start",
-                                                                           "type": "python",
-                                                                           "env_vars": {
-                                                                               "env1": "env1value"
-                                                                           }
-                                                                       }],
-                                                                       superliminal_after_tasks=[]))
+        expected = {
+            'pipeline': 'mypipe',
+            'tasks': [
+                {'env_vars': {'env1': 'env1value', 'env2': 'env2value'}, 'task': 'start', 'type': 'python'},
+                {'executor': 'emr', 'task': 'middle', 'task_param': 'task_middle_param', 'type': 'spark'},
+                {'env_vars': {'env2': 'env2value'}, 'task': 'end', 'task_param': 'task_end_param', 'type': 'python'},
+            ],
+        }
+        self.assertEqual(
+            expected,
+            default_configs.apply_task_defaults(
+                subliminal,
+                superliminal,
+                pipeline=pipeline,
+                superliminal_before_tasks=[{"task": "start", "type": "python", "env_vars": {"env1": "env1value"}}],
+                superliminal_after_tasks=[],
+            ),
+        )
 
     def test_missing_tasks_from_supr(self):
         pipeline = {
             'pipeline': 'mypipe',
             "tasks": [
-                {
-                    "task": "middle",
-                    "type": "spark",
-                    "task_param": "task_middle_param"
-                },
-                {
-                    "task": "end",
-                    "type": "python",
-                    "task_param": "task_end_param"
-                }
-            ]
+                {"task": "middle", "type": "spark", "task_param": "task_middle_param"},
+                {"task": "end", "type": "python", "task_param": "task_end_param"},
+            ],
         }
 
         subliminal = {'pipelines': [pipeline]}
 
         superliminal = {
             "task_defaults": {
-                "python": {
-                    "env_vars": {
-                        "env2": "env2value"
-                    }
-                },
-                "spark": {
-                    "task_param": "task_spark_param"
-                }
+                "python": {"env_vars": {"env2": "env2value"}},
+                "spark": {"task_param": "task_spark_param"},
             }
         }
 
-        expected = {'pipeline': 'mypipe',
-                    'tasks': [{'task': 'middle',
-                               'task_param': 'task_middle_param',
-                               'type': 'spark'},
-                              {'env_vars': {'env2': 'env2value'},
-                               'task': 'end',
-                               'task_param': 'task_end_param',
-                               'type': 'python'}]}
-
-        self.assertEqual(expected, default_configs.apply_task_defaults(subliminal,
-                                                                       superliminal,
-                                                                       pipeline=pipeline,
-                                                                       superliminal_before_tasks=[],
-                                                                       superliminal_after_tasks=[]))
+        expected = {
+            'pipeline': 'mypipe',
+            'tasks': [
+                {'task': 'middle', 'task_param': 'task_middle_param', 'type': 'spark'},
+                {'env_vars': {'env2': 'env2value'}, 'task': 'end', 'task_param': 'task_end_param', 'type': 'python'},
+            ],
+        }
+
+        self.assertEqual(
+            expected,
+            default_configs.apply_task_defaults(
+                subliminal, superliminal, pipeline=pipeline, superliminal_before_tasks=[], superliminal_after_tasks=[]
+            ),
+        )
 
     def test_missing_tasks_from_sub(self):
-        pipeline = {
-            'pipeline': 'mypipe'
-        }
+        pipeline = {'pipeline': 'mypipe'}
 
         subliminal = {'pipelines': [pipeline]}
 
         superliminal = {
             "task_defaults": {
-                "python": {
-                    "env_vars": {
-                        "env2": "env2value"
-                    }
-                },
+                "python": {"env_vars": {"env2": "env2value"}},
                 "spark": {
                     "task_param": "task_start_param",
-                }
+                },
             }
         }
 
-        expected = {'pipeline': 'mypipe',
-                    'tasks': [{
-                        'task': 'start',
-                        'task_param': 'task_middle_param',
-                        'type': 'spark'},
-                        {'env_vars': {'env2': 'env2value'},
-                         'task': 'end',
-                         'task_param': 'task_end_param',
-                         'type': 'end'}]}
-
-        self.assertEqual(expected,
-                         default_configs.apply_task_defaults(subliminal,
-                                                             superliminal,
-                                                             pipeline=pipeline,
-                                                             superliminal_before_tasks=[{
-                                                                 "task": "start",
-                                                                 "task_param": "task_middle_param",
-                                                                 "type": "spark"
-                                                             }],
-                                                             superliminal_after_tasks=[{
-                                                                 "env_vars": {
-                                                                     "env2": "env2value"
-                                                                 },
-                                                                 "task": "end",
-                                                                 "task_param": "task_end_param",
-                                                                 "type": "end"
-                                                             }]))
+        expected = {
+            'pipeline': 'mypipe',
+            'tasks': [
+                {'task': 'start', 'task_param': 'task_middle_param', 'type': 'spark'},
+                {'env_vars': {'env2': 'env2value'}, 'task': 'end', 'task_param': 'task_end_param', 'type': 'end'},
+            ],
+        }
+
+        self.assertEqual(
+            expected,
+            default_configs.apply_task_defaults(
+                subliminal,
+                superliminal,
+                pipeline=pipeline,
+                superliminal_before_tasks=[{"task": "start", "task_param": "task_middle_param", "type": "spark"}],
+                superliminal_after_tasks=[
+                    {"env_vars": {"env2": "env2value"}, "task": "end", "task_param": "task_end_param", "type": "end"}
+                ],
+            ),
+        )
diff --git a/tests/liminal/core/config/test_config.py b/tests/liminal/core/config/test_config.py
index 57201be..2a09122 100644
--- a/tests/liminal/core/config/test_config.py
+++ b/tests/liminal/core/config/test_config.py
@@ -24,49 +24,31 @@ from liminal.core.config.config import ConfigUtil
 
 # noinspection PyUnresolvedReferences,DuplicatedCode
 class TestConfigUtil(TestCase):
-
     @mock.patch('liminal.core.util.files_util.load')
     def test_safe_load(self, find_config_files_mock):
         subliminal = {
             'name': 'my_subliminal_test',
             'type': 'sub',
             'super': 'my_superliminal_test',
-            'images': [{
-                'image': 'my_image'
-            }],
-            'pipelines': [
-                {'name': 'mypipe1', 'param': 'constant'},
-                {'name': 'mypipe2', 'param': 'constant'}
-            ],
-            'pipeline_defaults': {
-                'param1': 'param1_value'
-            },
-            'task_defaults': {
-                'job_start': {
-                    'task_sub_def': 'task_sub_def_value'
-                }
-            }
+            'images': [{'image': 'my_image'}],
+            'pipelines': [{'name': 'mypipe1', 'param': 'constant'}, {'name': 'mypipe2', 'param': 'constant'}],
+            'pipeline_defaults': {'param1': 'param1_value'},
+            'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
         }
         superliminal = {
             'name': 'my_superliminal_test',
             'type': 'super',
             'super': 'super_superliminal',
-            'images': [{
-                'image': 'my_image',
-                'source': '.'
-            }],
-            'pipeline_defaults': {
-                'param2': 'param2super_value',
-                'param3': 'param3super_value'
-            },
+            'images': [{'image': 'my_image', 'source': '.'}],
+            'pipeline_defaults': {'param2': 'param2super_value', 'param3': 'param3super_value'},
             'task_defaults': {
                 'job_start': {
                     'task_def1': 'task_def1_value',
                     'task_def2': {
                         'task_def2_1': 'task_def2_1_value',
-                    }
+                    },
                 }
-            }
+            },
         }
         super_superliminal = {
             'name': 'super_superliminal',
@@ -74,97 +56,127 @@ class TestConfigUtil(TestCase):
             'pipeline_defaults': {
                 'param2': 'param2super_value',
                 'param3': 'param3hyper_value',
-                'param4': 'param4hyper_value'
-            }
+                'param4': 'param4hyper_value',
+            },
         }
 
-        expected = [{
-            'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
-            'name': 'my_subliminal_test',
-            'pipeline_defaults': {'param1': 'param1_value'},
-            'pipelines': [{'description': 'add defaults parameters for all pipelines',
-                           'name': 'mypipe1',
-                           'param': 'constant',
-                           'param1': 'param1_value',
-                           'param2': 'param2super_value',
-                           'param3': 'param3super_value',
-                           'param4': 'param4hyper_value',
-                           'tasks': [{'task': 'start',
-                                      'task_def1': 'task_def1_value',
-                                      'task_def2': {'task_def2_1': 'task_def2_1_value'},
-                                      'task_sub_def': 'task_sub_def_value',
-                                      'type': 'job_start'},
-                                     {'task': 'end', 'type': 'job_end'}]},
-                          {'description': 'add defaults parameters for all pipelines',
-                           'name': 'mypipe2',
-                           'param': 'constant',
-                           'param1': 'param1_value',
-                           'param2': 'param2super_value',
-                           'param3': 'param3super_value',
-                           'param4': 'param4hyper_value',
-                           'tasks': [{'task': 'start',
-                                      'task_def1': 'task_def1_value',
-                                      'task_def2': {'task_def2_1': 'task_def2_1_value'},
-                                      'task_sub_def': 'task_sub_def_value',
-                                      'type': 'job_start'},
-                                     {'task': 'end', 'type': 'job_end'}]}],
-            'service_defaults': {'description': 'add defaults parameters for all '
-                                                'services'},
-            'images': [{'image': 'my_image', 'source': '.'}],
-            'services': [],
-            'super': 'my_superliminal_test',
-            'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
-            'type': 'sub'
-        }]
-
-        expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
-                                   {'executor': 'airflow_executor', 'type': 'airflow'}],
-                     'images': [{'image': 'my_image', 'source': '.'}],
-                     'name': 'my_subliminal_test',
-                     'pipeline_defaults': {'param1': 'param1_value'},
-                     'pipelines': [{'description': 'add defaults parameters for all pipelines',
-                                    'name': 'mypipe1',
-                                    'param': 'constant',
-                                    'param1': 'param1_value',
-                                    'param2': 'param2super_value',
-                                    'param3': 'param3super_value',
-                                    'param4': 'param4hyper_value',
-                                    'tasks': [{'executor': 'airflow_executor',
-                                               'task': 'start',
-                                               'task_def1': 'task_def1_value',
-                                               'task_def2': {'task_def2_1': 'task_def2_1_value'},
-                                               'task_sub_def': 'task_sub_def_value',
-                                               'type': 'job_start'},
-                                              {'executor': 'airflow_executor',
-                                               'task': 'end',
-                                               'type': 'job_end'}]},
-                                   {'description': 'add defaults parameters for all pipelines',
-                                    'name': 'mypipe2',
-                                    'param': 'constant',
-                                    'param1': 'param1_value',
-                                    'param2': 'param2super_value',
-                                    'param3': 'param3super_value',
-                                    'param4': 'param4hyper_value',
-                                    'tasks': [{'executor': 'airflow_executor',
-                                               'task': 'start',
-                                               'task_def1': 'task_def1_value',
-                                               'task_def2': {'task_def2_1': 'task_def2_1_value'},
-                                               'task_sub_def': 'task_sub_def_value',
-                                               'type': 'job_start'},
-                                              {'executor': 'airflow_executor',
-                                               'task': 'end',
-                                               'type': 'job_end'}]}],
-                     'service_defaults': {'description': 'add defaults parameters for all '
-                                                         'services'},
-                     'services': [],
-                     'super': 'my_superliminal_test',
-                     'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
-                     'type': 'sub'}]
+        expected = [
+            {
+                'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'}],
+                'name': 'my_subliminal_test',
+                'pipeline_defaults': {'param1': 'param1_value'},
+                'pipelines': [
+                    {
+                        'description': 'add defaults parameters for all pipelines',
+                        'name': 'mypipe1',
+                        'param': 'constant',
+                        'param1': 'param1_value',
+                        'param2': 'param2super_value',
+                        'param3': 'param3super_value',
+                        'param4': 'param4hyper_value',
+                        'tasks': [
+                            {
+                                'task': 'start',
+                                'task_def1': 'task_def1_value',
+                                'task_def2': {'task_def2_1': 'task_def2_1_value'},
+                                'task_sub_def': 'task_sub_def_value',
+                                'type': 'job_start',
+                            },
+                            {'task': 'end', 'type': 'job_end'},
+                        ],
+                    },
+                    {
+                        'description': 'add defaults parameters for all pipelines',
+                        'name': 'mypipe2',
+                        'param': 'constant',
+                        'param1': 'param1_value',
+                        'param2': 'param2super_value',
+                        'param3': 'param3super_value',
+                        'param4': 'param4hyper_value',
+                        'tasks': [
+                            {
+                                'task': 'start',
+                                'task_def1': 'task_def1_value',
+                                'task_def2': {'task_def2_1': 'task_def2_1_value'},
+                                'task_sub_def': 'task_sub_def_value',
+                                'type': 'job_start',
+                            },
+                            {'task': 'end', 'type': 'job_end'},
+                        ],
+                    },
+                ],
+                'service_defaults': {'description': 'add defaults parameters for all ' 'services'},
+                'images': [{'image': 'my_image', 'source': '.'}],
+                'services': [],
+                'super': 'my_superliminal_test',
+                'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
+                'type': 'sub',
+            }
+        ]
+
+        expected = [
+            {
+                'executors': [
+                    {'executor': 'default_k8s', 'type': 'kubernetes'},
+                    {'executor': 'airflow_executor', 'type': 'airflow'},
+                ],
+                'images': [{'image': 'my_image', 'source': '.'}],
+                'name': 'my_subliminal_test',
+                'pipeline_defaults': {'param1': 'param1_value'},
+                'pipelines': [
+                    {
+                        'description': 'add defaults parameters for all pipelines',
+                        'name': 'mypipe1',
+                        'param': 'constant',
+                        'param1': 'param1_value',
+                        'param2': 'param2super_value',
+                        'param3': 'param3super_value',
+                        'param4': 'param4hyper_value',
+                        'tasks': [
+                            {
+                                'executor': 'airflow_executor',
+                                'task': 'start',
+                                'task_def1': 'task_def1_value',
+                                'task_def2': {'task_def2_1': 'task_def2_1_value'},
+                                'task_sub_def': 'task_sub_def_value',
+                                'type': 'job_start',
+                            },
+                            {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'},
+                        ],
+                    },
+                    {
+                        'description': 'add defaults parameters for all pipelines',
+                        'name': 'mypipe2',
+                        'param': 'constant',
+                        'param1': 'param1_value',
+                        'param2': 'param2super_value',
+                        'param3': 'param3super_value',
+                        'param4': 'param4hyper_value',
+                        'tasks': [
+                            {
+                                'executor': 'airflow_executor',
+                                'task': 'start',
+                                'task_def1': 'task_def1_value',
+                                'task_def2': {'task_def2_1': 'task_def2_1_value'},
+                                'task_sub_def': 'task_sub_def_value',
+                                'type': 'job_start',
+                            },
+                            {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'},
+                        ],
+                    },
+                ],
+                'service_defaults': {'description': 'add defaults parameters for all ' 'services'},
+                'services': [],
+                'super': 'my_superliminal_test',
+                'task_defaults': {'job_start': {'task_sub_def': 'task_sub_def_value'}},
+                'type': 'sub',
+            }
+        ]
 
         find_config_files_mock.return_value = {
             'my_subliminal_test': subliminal,
             'my_superliminal_test': superliminal,
-            'super_superliminal': super_superliminal
+            'super_superliminal': super_superliminal,
         }
 
         config_util = ConfigUtil('')
@@ -177,62 +189,50 @@ class TestConfigUtil(TestCase):
     @mock.patch('liminal.core.util.files_util.load')
     def test_get_config(self, find_config_files_mock):
         find_config_files_mock.return_value = {
-            'my_subliminal_test': {
-                'type': 'sub'
-            },
-            'my_superliminal_test': {
-                'type': 'super'
-            }
+            'my_subliminal_test': {'type': 'sub'},
+            'my_superliminal_test': {'type': 'super'},
         }
 
         config_util = ConfigUtil('')
 
-        self.assertEqual({'type': 'sub'},
-                         config_util._ConfigUtil__get_config('my_subliminal_test'))
+        self.assertEqual({'type': 'sub'}, config_util._ConfigUtil__get_config('my_subliminal_test'))
 
-        self.assertEqual({'type': 'super'},
-                         config_util._ConfigUtil__get_config('my_superliminal_test'))
+        self.assertEqual({'type': 'super'}, config_util._ConfigUtil__get_config('my_superliminal_test'))
 
     @mock.patch('liminal.core.util.files_util.load')
     def test_get_superliminal(self, find_config_files_mock):
-        base = {'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
-                              {'executor': 'airflow_executor', 'type': 'airflow'}],
-                'name': 'base',
-                'pipeline_defaults': {'after_tasks': [{'task': 'end', 'type': 'job_end'}],
-                                      'before_tasks': [{'task': 'start', 'type': 'job_start'}],
-                                      'description': 'add defaults parameters for all '
-                                                     'pipelines'},
-                'service_defaults': {'description': 'add defaults parameters for all '
-                                                    'services'},
-                'task_defaults': {'description': 'add defaults parameters for all tasks '
-                                                 'separate by task type',
-                                  'job_end': {'executor': 'airflow_executor'},
-                                  'job_start': {'executor': 'airflow_executor'},
-                                  'python': {'executor': 'default_k8s'},
-                                  'spark': {'executor': 'default_k8s'}},
-                'type': 'super'}
-        subliminal = {
-            'name': 'subliminal_test',
-            'type': 'sub'
+        base = {
+            'executors': [
+                {'executor': 'default_k8s', 'type': 'kubernetes'},
+                {'executor': 'airflow_executor', 'type': 'airflow'},
+            ],
+            'name': 'base',
+            'pipeline_defaults': {
+                'after_tasks': [{'task': 'end', 'type': 'job_end'}],
+                'before_tasks': [{'task': 'start', 'type': 'job_start'}],
+                'description': 'add defaults parameters for all ' 'pipelines',
+            },
+            'service_defaults': {'description': 'add defaults parameters for all ' 'services'},
+            'task_defaults': {
+                'description': 'add defaults parameters for all tasks ' 'separate by task type',
+                'job_end': {'executor': 'airflow_executor'},
+                'job_start': {'executor': 'airflow_executor'},
+                'python': {'executor': 'default_k8s'},
+                'spark': {'executor': 'default_k8s'},
+            },
+            'type': 'super',
         }
+        subliminal = {'name': 'subliminal_test', 'type': 'sub'}
 
-        find_config_files_mock.return_value = {
-            'subliminal_test': subliminal
-        }
+        find_config_files_mock.return_value = {'subliminal_test': subliminal}
 
         config_util = ConfigUtil('')
 
-        self.assertEqual(base,
-                         config_util._ConfigUtil__get_superliminal(subliminal, False))
+        self.assertEqual(base, config_util._ConfigUtil__get_superliminal(subliminal, False))
 
-        self.assertEqual({},
-                         config_util._ConfigUtil__get_superliminal(base, False))
+        self.assertEqual({}, config_util._ConfigUtil__get_superliminal(base, False))
 
-        liminal = {
-            'name': 'subliminal_test',
-            'type': 'sub',
-            'super': 'my_superliminal'
-        }
+        liminal = {'name': 'subliminal_test', 'type': 'sub', 'super': 'my_superliminal'}
 
         with self.assertRaises(FileNotFoundError):
             config_util._ConfigUtil__get_superliminal(liminal, False)
@@ -247,48 +247,38 @@ class TestConfigUtil(TestCase):
                 'before_tasks': [
                     {'task': 'start-2', 'type': 'spark'},
                 ],
-                'after_tasks': [
-                    {'task': 'end-1', 'type': 'spark'}
-                ]
+                'after_tasks': [{'task': 'end-1', 'type': 'spark'}],
             },
-            'task_defaults': {
-                'task_def1': 'task_def1_value'
-            }
+            'task_defaults': {'task_def1': 'task_def1_value'},
         }
 
         super_superliminal = {
             'name': 'super_superliminal',
             'type': 'super',
             'pipeline_defaults': {
-                'before_tasks': [
-                    {'task': 'start-1', 'type': 'spark'}],
-                'after_tasks': [
-                    {'task': 'end-2', 'type': 'spark'}
-                ]
-            }
+                'before_tasks': [{'task': 'start-1', 'type': 'spark'}],
+                'after_tasks': [{'task': 'end-2', 'type': 'spark'}],
+            },
         }
 
         config_util = ConfigUtil('')
 
-        find_config_files_mock.return_value = {
-            'super_superliminal': super_superliminal,
-            'superliminal': superliminal
-        }
+        find_config_files_mock.return_value = {'super_superliminal': super_superliminal, 'superliminal': superliminal}
 
         expected = {
             'name': 'my_superliminal_test',
-            'pipeline_defaults': {'after_tasks': [{'task': 'end-1', 'type': 'spark'},
-                                                  {'task': 'end-2', 'type': 'spark'}],
-                                  'before_tasks': [{'task': 'start-1', 'type': 'spark'},
-                                                   {'task': 'start-2', 'type': 'spark'}]},
+            'pipeline_defaults': {
+                'after_tasks': [{'task': 'end-1', 'type': 'spark'}, {'task': 'end-2', 'type': 'spark'}],
+                'before_tasks': [{'task': 'start-1', 'type': 'spark'}, {'task': 'start-2', 'type': 'spark'}],
+            },
             'super': 'super_superliminal',
             'task_defaults': {'task_def1': 'task_def1_value'},
-            'type': 'super'
+            'type': 'super',
         }
 
-        self.assertEqual(expected,
-                         dict(config_util._ConfigUtil__merge_superliminals(superliminal,
-                                                                           super_superliminal)))
+        self.assertEqual(
+            expected, dict(config_util._ConfigUtil__merge_superliminals(superliminal, super_superliminal))
+        )
 
     @mock.patch('liminal.core.util.files_util.load')
     @mock.patch.dict(os.environ, {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True'})
@@ -304,40 +294,31 @@ class TestConfigUtil(TestCase):
                 'image': 'prod image',
                 'a': '{{env}}1',
                 'b': '{{a}}2',
-                'c': '{{a}}{{b}}2'
+                'c': '{{a}}{{b}}2',
             },
             'pipelines': [
-                {'name': 'mypipe1', 'param': '{{var}}',
-                 'tasks': [
-                     {'task': 'sub_tasks',
-                      'type': 'dummy'},
-                 ]},
-                {'name': 'mypipe2', 'param': '{{var-2   }}', 'tasks': [
-                    {'task': 'sub_tasks',
-                     'type': 'dummy'},
-                ]}
-            ],
-            'pipeline_defaults': {
-                'param1': '{{var-2}}'
-            },
-            'task_defaults': {
-                'job_start': {
-                    'task_def1:': 'task_sub_def_value'
-                }
-
-            },
-            'services': [
                 {
-                    'name': 'my_python_server',
-                    'type': 'python_server',
-                    'image': '{{image}}'
+                    'name': 'mypipe1',
+                    'param': '{{var}}',
+                    'tasks': [
+                        {'task': 'sub_tasks', 'type': 'dummy'},
+                    ],
                 },
                 {
-                    'name': 'my_python_server_for_stg',
-                    'type': 'python_server',
-                    'image': '{{default_image}}'
-                }
-            ]}
+                    'name': 'mypipe2',
+                    'param': '{{var-2   }}',
+                    'tasks': [
+                        {'task': 'sub_tasks', 'type': 'dummy'},
+                    ],
+                },
+            ],
+            'pipeline_defaults': {'param1': '{{var-2}}'},
+            'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
+            'services': [
+                {'name': 'my_python_server', 'type': 'python_server', 'image': '{{image}}'},
+                {'name': 'my_python_server_for_stg', 'type': 'python_server', 'image': '{{default_image}}'},
+            ],
+        }
 
         superliminal = {
             'name': 'my_superliminal_test',
@@ -346,7 +327,7 @@ class TestConfigUtil(TestCase):
                 'var-2': 'override',
                 'var3': 'super_var',
                 'default_image': 'default_image_value',
-                'image': 'default_image_value'
+                'image': 'default_image_value',
             },
             'super': 'super_superliminal',
             'pipeline_defaults': {
@@ -354,7 +335,7 @@ class TestConfigUtil(TestCase):
                 'param3': 'param3super_value',
                 'before_tasks': [
                     {'task': 'second_task', 'type': 'dummy'},
-                ]
+                ],
             },
             'task_defaults': {
                 'pipeline': {
@@ -362,16 +343,14 @@ class TestConfigUtil(TestCase):
                     'task_def1': 'task_def1_value',
                     'task_def2': {
                         'task_def2_1': 'task_def2_1_value',
-                    }
+                    },
                 }
-            }
+            },
         }
         super_superliminal = {
             'name': 'super_superliminal',
             'type': 'super',
-            'variables': {
-                'default_image': 'def_default_image_value'
-            },
+            'variables': {'default_image': 'def_default_image_value'},
             'pipeline_defaults': {
                 'global_conf': '{{var3}}',
                 'param2': 'param2super_value',
@@ -379,76 +358,99 @@ class TestConfigUtil(TestCase):
                 'param4': 'param4hyper_value',
                 'after_tasks': [
                     {'task': 'before_last_task', 'type': 'dummy'},
-                ]
-            }
+                ],
+            },
         }
 
-        expected = [{'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
-                                   {'executor': 'airflow_executor', 'type': 'airflow'}],
-                     'name': 'my_subliminal_test',
-                     'pipeline_defaults': {'param1': '-case'},
-                     'pipelines': [{'description': 'add defaults parameters for all pipelines',
-                                    'global_conf': 'super_var',
-                                    'name': 'mypipe1',
-                                    'param': 'simple case',
-                                    'param1': '-case',
-                                    'param2': '{{pipe-var}}',
-                                    'param3': 'param3super_value',
-                                    'param4': 'param4hyper_value',
-                                    'tasks': [{'executor': 'airflow_executor',
-                                               'task': 'start',
-                                               'task_def1:': 'task_sub_def_value',
-                                               'type': 'job_start'},
-                                              {'task': 'second_task', 'type': 'dummy'},
-                                              {'task': 'sub_tasks', 'type': 'dummy'},
-                                              {'task': 'before_last_task', 'type': 'dummy'},
-                                              {'executor': 'airflow_executor',
-                                               'task': 'end',
-                                               'type': 'job_end'}]},
-                                   {'description': 'add defaults parameters for all pipelines',
-                                    'global_conf': 'super_var',
-                                    'name': 'mypipe2',
-                                    'param': '-case',
-                                    'param1': '-case',
-                                    'param2': '{{pipe-var}}',
-                                    'param3': 'param3super_value',
-                                    'param4': 'param4hyper_value',
-                                    'tasks': [{'executor': 'airflow_executor',
-                                               'task': 'start',
-                                               'task_def1:': 'task_sub_def_value',
-                                               'type': 'job_start'},
-                                              {'task': 'second_task', 'type': 'dummy'},
-                                              {'task': 'sub_tasks', 'type': 'dummy'},
-                                              {'task': 'before_last_task', 'type': 'dummy'},
-                                              {'executor': 'airflow_executor',
-                                               'task': 'end',
-                                               'type': 'job_end'}]}],
-                     'service_defaults': {'description': 'add defaults parameters for all '
-                                                         'services'},
-                     'images': [],
-                     'services': [{'description': 'add defaults parameters for all services',
-                                   'image': 'prod image',
-                                   'name': 'my_python_server',
-                                   'type': 'python_server'},
-                                  {'description': 'add defaults parameters for all services',
-                                   'image': 'default_image_value',
-                                   'name': 'my_python_server_for_stg',
-                                   'type': 'python_server'}],
-                     'super': 'my_superliminal_test',
-                     'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
-                     'type': 'sub',
-                     'variables': {'a': 'myenv1',
-                                   'b': 'myenv12',
-                                   'c': 'myenv1myenv122',
-                                   'image': 'prod image',
-                                   'var': 'simple case',
-                                   'var-2': '-case',
-                                   'var_2': '_case'}}]
+        expected = [
+            {
+                'executors': [
+                    {'executor': 'default_k8s', 'type': 'kubernetes'},
+                    {'executor': 'airflow_executor', 'type': 'airflow'},
+                ],
+                'name': 'my_subliminal_test',
+                'pipeline_defaults': {'param1': '-case'},
+                'pipelines': [
+                    {
+                        'description': 'add defaults parameters for all pipelines',
+                        'global_conf': 'super_var',
+                        'name': 'mypipe1',
+                        'param': 'simple case',
+                        'param1': '-case',
+                        'param2': '{{pipe-var}}',
+                        'param3': 'param3super_value',
+                        'param4': 'param4hyper_value',
+                        'tasks': [
+                            {
+                                'executor': 'airflow_executor',
+                                'task': 'start',
+                                'task_def1:': 'task_sub_def_value',
+                                'type': 'job_start',
+                            },
+                            {'task': 'second_task', 'type': 'dummy'},
+                            {'task': 'sub_tasks', 'type': 'dummy'},
+                            {'task': 'before_last_task', 'type': 'dummy'},
+                            {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'},
+                        ],
+                    },
+                    {
+                        'description': 'add defaults parameters for all pipelines',
+                        'global_conf': 'super_var',
+                        'name': 'mypipe2',
+                        'param': '-case',
+                        'param1': '-case',
+                        'param2': '{{pipe-var}}',
+                        'param3': 'param3super_value',
+                        'param4': 'param4hyper_value',
+                        'tasks': [
+                            {
+                                'executor': 'airflow_executor',
+                                'task': 'start',
+                                'task_def1:': 'task_sub_def_value',
+                                'type': 'job_start',
+                            },
+                            {'task': 'second_task', 'type': 'dummy'},
+                            {'task': 'sub_tasks', 'type': 'dummy'},
+                            {'task': 'before_last_task', 'type': 'dummy'},
+                            {'executor': 'airflow_executor', 'task': 'end', 'type': 'job_end'},
+                        ],
+                    },
+                ],
+                'service_defaults': {'description': 'add defaults parameters for all ' 'services'},
+                'images': [],
+                'services': [
+                    {
+                        'description': 'add defaults parameters for all services',
+                        'image': 'prod image',
+                        'name': 'my_python_server',
+                        'type': 'python_server',
+                    },
+                    {
+                        'description': 'add defaults parameters for all services',
+                        'image': 'default_image_value',
+                        'name': 'my_python_server_for_stg',
+                        'type': 'python_server',
+                    },
+                ],
+                'super': 'my_superliminal_test',
+                'task_defaults': {'job_start': {'task_def1:': 'task_sub_def_value'}},
+                'type': 'sub',
+                'variables': {
+                    'a': 'myenv1',
+                    'b': 'myenv12',
+                    'c': 'myenv1myenv122',
+                    'image': 'prod image',
+                    'var': 'simple case',
+                    'var-2': '-case',
+                    'var_2': '_case',
+                },
+            }
+        ]
 
         find_config_files_mock.return_value = {
             'my_subliminal_test': subliminal,
             'my_superliminal_test': superliminal,
-            'super_superliminal': super_superliminal
+            'super_superliminal': super_superliminal,
         }
 
         config_util = ConfigUtil('')
@@ -462,49 +464,61 @@ class TestConfigUtil(TestCase):
     @mock.patch('liminal.core.environment.get_airflow_home_dir')
     @mock.patch('liminal.core.util.files_util.load')
     @mock.patch.dict(os.environ, {'LIMINAL_STAND_ALONE_MODE': 'True', 'POD_NAMESPACE': 'my_pod_ns'})
-    def test_liminal_config_snapshot(self, find_config_files_mock,
-                                     get_airflow_dir_mock, path_exists_mock):
+    def test_liminal_config_snapshot(self, find_config_files_mock, get_airflow_dir_mock, path_exists_mock):
         subliminal = {
             'name': 'my_subliminal_test',
             'type': 'sub',
-            'variables': {
-                'var': 1,
-                'var-2': True
-            },
-            'pipelines': [
-                {'name': 'mypipe1', 'param': '{{var}}'},
-                {'name': 'mypipe2', 'param': '{{var-2   }}'}
-            ]
+            'variables': {'var': 1, 'var-2': True},
+            'pipelines': [{'name': 'mypipe1', 'param': '{{var}}'}, {'name': 'mypipe2', 'param': '{{var-2   }}'}],
         }
 
-        expected = {'name': 'my_subliminal_test', 'type': 'sub',
-                    'executors': [{'executor': 'default_k8s', 'type': 'kubernetes'},
-                                  {'executor': 'airflow_executor', 'type': 'airflow'}],
-                    'service_defaults': {'description': 'add defaults parameters for all services'},
-                    'task_defaults': {
-                        'description': 'add defaults parameters for all tasks separate by task type',
-                        'python': {'executor': 'default_k8s'}, 'spark': {'executor': 'default_k8s'},
-                        'job_end': {'executor': 'airflow_executor'},
-                        'job_start': {'executor': 'airflow_executor'}}, 'pipeline_defaults': {
+        expected = {
+            'name': 'my_subliminal_test',
+            'type': 'sub',
+            'executors': [
+                {'executor': 'default_k8s', 'type': 'kubernetes'},
+                {'executor': 'airflow_executor', 'type': 'airflow'},
+            ],
+            'service_defaults': {'description': 'add defaults parameters for all services'},
+            'task_defaults': {
+                'description': 'add defaults parameters for all tasks separate by task type',
+                'python': {'executor': 'default_k8s'},
+                'spark': {'executor': 'default_k8s'},
+                'job_end': {'executor': 'airflow_executor'},
+                'job_start': {'executor': 'airflow_executor'},
+            },
+            'pipeline_defaults': {
                 'description': 'add defaults parameters for all pipelines',
                 'before_tasks': [{'task': 'start', 'type': 'job_start'}],
-                'after_tasks': [{'task': 'end', 'type': 'job_end'}]},
-                    'variables': {'var': 1, 'var-2': True}, 'pipelines': [
-                {'name': 'mypipe1', 'param': '1',
-                 'description': 'add defaults parameters for all pipelines',
-                 'tasks': [{'task': 'start', 'type': 'job_start', 'executor': 'airflow_executor'},
-                           {'task': 'end', 'type': 'job_end', 'executor': 'airflow_executor'}]},
-                {'name': 'mypipe2', 'param': 'True',
-                 'description': 'add defaults parameters for all pipelines',
-                 'tasks': [{'task': 'start', 'type': 'job_start', 'executor': 'airflow_executor'},
-                           {'task': 'end', 'type': 'job_end', 'executor': 'airflow_executor'}]}],
-                    'images': [],
-                    'services': []}
-
-        find_config_files_mock.return_value = {
-            'my_subliminal_test': subliminal
+                'after_tasks': [{'task': 'end', 'type': 'job_end'}],
+            },
+            'variables': {'var': 1, 'var-2': True},
+            'pipelines': [
+                {
+                    'name': 'mypipe1',
+                    'param': '1',
+                    'description': 'add defaults parameters for all pipelines',
+                    'tasks': [
+                        {'task': 'start', 'type': 'job_start', 'executor': 'airflow_executor'},
+                        {'task': 'end', 'type': 'job_end', 'executor': 'airflow_executor'},
+                    ],
+                },
+                {
+                    'name': 'mypipe2',
+                    'param': 'True',
+                    'description': 'add defaults parameters for all pipelines',
+                    'tasks': [
+                        {'task': 'start', 'type': 'job_start', 'executor': 'airflow_executor'},
+                        {'task': 'end', 'type': 'job_end', 'executor': 'airflow_executor'},
+                    ],
+                },
+            ],
+            'images': [],
+            'services': [],
         }
 
+        find_config_files_mock.return_value = {'my_subliminal_test': subliminal}
+
         get_airflow_dir_mock.return_value = '/tmp'
         path_exists_mock.return_value = True
 
@@ -514,34 +528,21 @@ class TestConfigUtil(TestCase):
                 config_util.safe_load(is_render_variables=True)
                 config_util.snapshot_final_liminal_configs()
 
-                m.assert_called_once_with(
-                    os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w')
+                m.assert_called_once_with(os.path.join('/tmp', '../liminal_config_files/my_subliminal_test.yml'), 'w')
                 ydm.assert_called_once_with(expected, m.return_value, default_flow_style=False)
 
     @mock.patch('liminal.core.util.files_util.load')
     def test_soft_merge_load(self, find_config_files_mock):
-        subliminal = {
-            'name': 'my_name',
-            'type': 'sub',
-            'super': 'my_super'
-        }
+        subliminal = {'name': 'my_name', 'type': 'sub', 'super': 'my_super'}
         find_config_files_mock.return_value = {'my_subliminal_test': subliminal}
 
         config_util = ConfigUtil('')
 
-        self.assertEqual([subliminal],
-                         config_util.safe_load(is_render_variables=True, soft_merge=True))
+        self.assertEqual([subliminal], config_util.safe_load(is_render_variables=True, soft_merge=True))
 
     def test_non_soft_merge_load(self):
-        subliminal = {
-            'name': 'my_name',
-            'type': 'sub',
-            'super': 'my_super'
-        }
+        subliminal = {'name': 'my_name', 'type': 'sub', 'super': 'my_super'}
 
         config_util = ConfigUtil('')
 
-        self.assertRaises(FileNotFoundError,
-                          config_util._ConfigUtil__get_superliminal,
-                          subliminal,
-                          False)
+        self.assertRaises(FileNotFoundError, config_util._ConfigUtil__get_superliminal, subliminal, False)
diff --git a/tests/liminal/core/util/test_dict_utils.py b/tests/liminal/core/util/test_dict_utils.py
index e5c2b10..f387724 100644
--- a/tests/liminal/core/util/test_dict_utils.py
+++ b/tests/liminal/core/util/test_dict_utils.py
@@ -24,44 +24,17 @@ from liminal.core.util import dict_util
 
 class TestDictUtils(TestCase):
     def setUp(self) -> None:
-        self.dict1 = {
-            "env": "env1",
-            "env_dict":
-                {
-                    "env3": "env3"
-                },
-            "env4": "env4"
-        }
+        self.dict1 = {"env": "env1", "env_dict": {"env3": "env3"}, "env4": "env4"}
 
-        self.dict2 = {
-            "env": "env1",
-            "env_dict":
-                {
-                    "env2": "env2"
-                }
-        }
+        self.dict2 = {"env": "env1", "env_dict": {"env2": "env2"}}
 
     def test_merge_dicts(self):
-        expected = {
-            "env": "env1",
-            "env4": "env4",
-            "env_dict":
-                {
-                    "env2": "env2"
-                }
-        }
+        expected = {"env": "env1", "env4": "env4", "env_dict": {"env2": "env2"}}
 
         self.assertEqual(expected, dict_util.merge_dicts(self.dict1, self.dict2))
 
     def test_recursive_merge_dicts(self):
-        expected = {
-            "env": "env1",
-            "env4": "env4",
-            "env_dict": {
-                "env2": "env2",
-                "env3": "env3"
-            }
-        }
+        expected = {"env": "env1", "env4": "env4", "env_dict": {"env2": "env2", "env3": "env3"}}
         self.assertEqual(expected, dict_util.merge_dicts(self.dict1, self.dict2, True))
 
     def test_merge_with_empty(self):
@@ -74,30 +47,20 @@ class TestDictUtils(TestCase):
     def test_replace_variables_simple_case(self):
         dct = {
             "env": "{{env_var}}",
-            "env_dict":
-                {
-                    "env3": "{{ var1 }}"
-                },
+            "env_dict": {"env3": "{{ var1 }}"},
             "env4": "{{var2 }}",
             "env5": "{{{var2}}",
-            "env6": "{{var3}}"
+            "env6": "{{var3}}",
         }
 
-        variables = {
-            "env_var": "env value",
-            "var1": "value1",
-            "var2": "value2"
-        }
+        variables = {"env_var": "env value", "var1": "value1", "var2": "value2"}
 
         expected = {
             "env": "env value",
-            "env_dict":
-                {
-                    "env3": "value1"
-                },
+            "env_dict": {"env3": "value1"},
             "env4": "value2",
             "env5": "{value2",
-            "env6": "{{var3}}"
+            "env6": "{{var3}}",
         }
 
         self.assertEqual(expected, dict_util.replace_placeholders(dct, variables))
@@ -105,13 +68,10 @@ class TestDictUtils(TestCase):
     def test_replace_variables_empty_var(self):
         dct = {
             "env": "{{env_var}}",
-            "env_dict":
-                {
-                    "env3": "{{ var1 }}"
-                },
+            "env_dict": {"env3": "{{ var1 }}"},
             "env4": "{{var2 }}",
             "env5": "{{{var2}}",
-            "env6": "{{var3}}"
+            "env6": "{{var3}}",
         }
         self.assertEqual(dct, dict_util.replace_placeholders(dct, {}))
 
@@ -124,24 +84,23 @@ class TestDictUtils(TestCase):
         airflow_variable_mock.side_effect = airflow_variable_values
 
         dct = {
-            "query": "select * from my_table "
-                     "where event_type = {{event_type}} and region = {{region}}",
+            "query": "select * from my_table " "where event_type = {{event_type}} and region = {{region}}",
             "env": "{{prod}}, {{stg}}, {{playground}}",
-            "optional": "{{optionals}}"
+            "optional": "{{optionals}}",
         }
 
         variables = {
             "region": "us_east_1",
             "event_type": "subscription",
             "prod": "liminal production",
-            "stg": "liminal staging"
+            "stg": "liminal staging",
         }
 
         expected = {
-            'query': 'select * from my_table '
-                     'where event_type = subscription and region = us_east_1',
+            'query': 'select * from my_table ' 'where event_type = subscription and region = us_east_1',
             "env": "liminal production, liminal staging, liminal playground",
-            "optional": "{{optionals}}"}
+            "optional": "{{optionals}}",
+        }
 
         self.assertEqual(expected, dict_util.replace_placeholders(dct, variables))
 
@@ -154,16 +113,10 @@ class TestDictUtils(TestCase):
         airflow_variable_mock.side_effect = airflow_variable_values
 
         dct = {
-            "query": "select * from my_table "
-                     "where event_type = {{event_type}} and region = {{region}}",
+            "query": "select * from my_table " "where event_type = {{event_type}} and region = {{region}}",
             "env": ['{{prod}}', '{{stg}}', '{{playground}}'],
-            "tasks": [
-                {
-                    'id': 'id1',
-                    'image': '{{image}}'
-                }
-            ],
-            "optional": "{{optionals}}"
+            "tasks": [{'id': 'id1', 'image': '{{image}}'}],
+            "optional": "{{optionals}}",
         }
 
         variables = {
@@ -171,14 +124,15 @@ class TestDictUtils(TestCase):
             "event_type": "subscription",
             "prod": "liminal production",
             "stg": "liminal staging",
-            "image": "my_image_name"
+            "image": "my_image_name",
         }
 
-        expected = {'env': ['liminal production', 'liminal staging', 'liminal playground'],
-                    'optional': '{{optionals}}',
-                    'query': 'select * from my_table where event_type = subscription and region = '
-                             'us_east_1',
-                    'tasks': [{'id': 'id1', 'image': 'my_image_name'}]}
+        expected = {
+            'env': ['liminal production', 'liminal staging', 'liminal playground'],
+            'optional': '{{optionals}}',
+            'query': 'select * from my_table where event_type = subscription and region = ' 'us_east_1',
+            'tasks': [{'id': 'id1', 'image': 'my_image_name'}],
+        }
 
         self.assertEqual(expected, dict_util.replace_placeholders(dct, variables))
 
@@ -186,14 +140,16 @@ class TestDictUtils(TestCase):
     def test_replace_variables_from_env(self):
         dct = {
             "query": "select * from my_table "
-                     "where event_type = {{event_type}} and region = {{region}} from {{table}}"
+            "where event_type = {{event_type}} and region = {{region}} from {{table}}"
         }
 
         variables = {}
 
-        expected = {'query': 'select * from my_table '
-                             'where event_type = {{event_type}} '
-                             'and region = {{region}} from my_table'}
+        expected = {
+            'query': 'select * from my_table '
+            'where event_type = {{event_type}} '
+            'and region = {{region}} from my_table'
+        }
 
         self.assertEqual(expected, dict_util.replace_placeholders(dct, variables))
 
@@ -201,16 +157,16 @@ class TestDictUtils(TestCase):
     def test_replace_variables_from_variable_and_not_env(self):
         dct = {
             "query": "select * from my_table "
-                     "where event_type = {{event_type}} and region = {{region}} from {{table}}"
+            "where event_type = {{event_type}} and region = {{region}} from {{table}}"
         }
 
-        variables = {
-            "table": "my_variable_table"
-        }
+        variables = {"table": "my_variable_table"}
 
-        expected = {'query': 'select * from my_table '
-                             'where event_type = {{event_type}} '
-                             'and region = {{region}} from my_variable_table'}
+        expected = {
+            'query': 'select * from my_table '
+            'where event_type = {{event_type}} '
+            'and region = {{region}} from my_variable_table'
+        }
 
         self.assertEqual(expected, dict_util.replace_placeholders(dct, variables))
 
@@ -224,13 +180,15 @@ class TestDictUtils(TestCase):
 
         dct = {
             "query": "select * from my_table "
-                     "where event_type = {{event_type}} and region = {{region}} from {{table}}"
+            "where event_type = {{event_type}} and region = {{region}} from {{table}}"
         }
 
         variables = {}
 
-        expected = {'query': 'select * from my_table '
-                             'where event_type = {{event_type}} '
-                             'and region = {{region}} from my_airflow_table'}
+        expected = {
+            'query': 'select * from my_table '
+            'where event_type = {{event_type}} '
+            'and region = {{region}} from my_airflow_table'
+        }
 
         self.assertEqual(expected, dict_util.replace_placeholders(dct, variables))
diff --git a/tests/liminal/kubernetes/test_volume_util.py b/tests/liminal/kubernetes/test_volume_util.py
index d7a7192..36f6b8f 100644
--- a/tests/liminal/kubernetes/test_volume_util.py
+++ b/tests/liminal/kubernetes/test_volume_util.py
@@ -28,6 +28,7 @@ except Exception:
     msg = "Kubernetes is not running\n"
     sys.stdout.write(f"INFO: {msg}")
 
+
 class TestKubernetesVolume(unittest.TestCase):
     def setUp(self) -> None:
         self.config = {
@@ -35,31 +36,29 @@ class TestKubernetesVolume(unittest.TestCase):
                 {
                     'volume': 'gettingstartedvol-test',
                     'claim_name': 'gettingstartedvol-test-pvc',
-                    'local': {
-                        'path': '.'
-                        }
+                    'local': {'path': '.'},
                 }
             ]
         }
 
     def test_volume_config(self):
         volumes_config = volume_util.get_volume_configs(self.config, ".")
-        self.assertEqual(
-            str(self.config['volumes'][0]),
-            str(volumes_config[0]))
+        self.assertEqual(str(self.config['volumes'][0]), str(volumes_config[0]))
 
     def test_create_volume(self):
         self._delete_volumes()
         self._create_volumes()
         matching_volumes = volume_util._list_persistent_volumes(self.config['volumes'][0]['volume'])
-        self.assertTrue(self.config['volumes'][0]['volume'] in matching_volumes[0]['metadata']['name'],
-                        self.config['volumes'][0]['claim_name'] in matching_volumes[0]['spec']['claim_ref']['name'])
-                        
+        self.assertTrue(
+            self.config['volumes'][0]['volume'] in matching_volumes[0]['metadata']['name'],
+            self.config['volumes'][0]['claim_name'] in matching_volumes[0]['spec']['claim_ref']['name'],
+        )
+
     def test_delete_volume(self):
         self._create_volumes()
         self._delete_volumes()
         matching_volumes = volume_util._list_persistent_volumes(self.config['volumes'][0]['volume'])
-        self.assertEqual([],matching_volumes)
+        self.assertEqual([], matching_volumes)
 
     def _create_volumes(self):
         volume_util.create_local_volumes(self.config, ".")
diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
index ee40095..c82ca01 100644
--- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
+++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
@@ -34,7 +34,6 @@ IMAGE_NAME = 'liminal_server_image'
 
 
 class TestPythonServer(TestCase):
-
     def setUp(self) -> None:
         super().setUp()
         self.docker_client = docker.from_env()
@@ -49,15 +48,15 @@ class TestPythonServer(TestCase):
         versions = list(PythonImageVersions().supported_versions)
         for version in versions:
             build_out = self.__test_build_python_server(python_version=version)
-            self.assertTrue('RUN pip install -r requirements.txt' in build_out,
-                            'Incorrect pip command')
+            self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
 
     def test_build_python_server_with_pip_conf(self):
         build_out = self.__test_build_python_server(use_pip_conf=True)
 
         self.assertTrue(
             'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf  pip install' in build_out,
-            'Incorrect pip command')
+            'Incorrect pip command',
+        )
 
     def __test_build_python_server(self, use_pip_conf=False, python_version=None):
         base_path = os.path.join(os.path.dirname(__file__), '../../../liminal')
@@ -70,10 +69,9 @@ class TestPythonServer(TestCase):
         if python_version:
             config['python_version'] = python_version
 
-        builder = PythonServerImageBuilder(config=config,
-                                           base_path=base_path,
-                                           relative_source_path='myserver',
-                                           tag=IMAGE_NAME)
+        builder = PythonServerImageBuilder(
+            config=config, base_path=base_path, relative_source_path='myserver', tag=IMAGE_NAME
+        )
 
         build_out = str(builder.build())
 
@@ -88,10 +86,11 @@ class TestPythonServer(TestCase):
 
         encoding = 'ascii'
 
-        server_response = str(urllib.request.urlopen(
-            'http://localhost:9294/myendpoint1',
-            data=json_string.encode(encoding)
-        ).read().decode(encoding))
+        server_response = str(
+            urllib.request.urlopen('http://localhost:9294/myendpoint1', data=json_string.encode(encoding))
+            .read()
+            .decode(encoding)
+        )
 
         logging.info(f'Response from server: {server_response}')
 
@@ -117,15 +116,12 @@ class TestPythonServer(TestCase):
             matching_containers = self.__get_docker_containers()
 
     def __get_docker_containers(self):
-        return self.docker_client.containers.list(
-            filters={'ancestor': IMAGE_NAME}
-        )
+        return self.docker_client.containers.list(filters={'ancestor': IMAGE_NAME})
 
     def __run_container(self):
         try:
             logging.info(f'Running container for image: {IMAGE_NAME}')
-            self.docker_client.containers.run(IMAGE_NAME, ports={'80/tcp': 9294},
-                                              detach=True)
+            self.docker_client.containers.run(IMAGE_NAME, ports={'80/tcp': 9294}, detach=True)
         except Exception as err:
             logging.exception(err)
             pass
@@ -140,13 +136,7 @@ class TestPythonServer(TestCase):
             'input_path': 'my_input',
             'output_path': '/my_output.json',
             'no_cache': True,
-            'endpoints': [
-                {
-                    'endpoint': '/myendpoint1',
-                    'module': 'my_server',
-                    'function': 'myendpoint1func'
-                }
-            ]
+            'endpoints': [{'endpoint': '/myendpoint1', 'module': 'my_server', 'function': 'myendpoint1func'}],
         }
 
 
diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py
index 1046a25..94686a3 100644
--- a/tests/runners/airflow/build/python/test_python_image_builder.py
+++ b/tests/runners/airflow/build/python/test_python_image_builder.py
@@ -43,11 +43,9 @@ class TestPythonImageBuilder(TestCase):
         self.__remove_dir(self.temp_airflow_dir)
 
     def test_build(self):
-        for python_version in [None,
-                               PythonImageVersions().supported_versions[0]]:
+        for python_version in [None, PythonImageVersions().supported_versions[0]]:
             build_out = self.__test_build(python_version=python_version)
-        self.assertTrue('RUN pip install -r requirements.txt' in build_out,
-                        'Incorrect pip command')
+        self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
 
         self.__test_image()
 
@@ -56,7 +54,8 @@ class TestPythonImageBuilder(TestCase):
 
         self.assertTrue(
             'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf  pip install' in build_out,
-            'Incorrect pip command')
+            'Incorrect pip command',
+        )
 
         self.__test_image()
 
@@ -75,10 +74,9 @@ class TestPythonImageBuilder(TestCase):
         if python_version:
             config['python_version'] = python_version
 
-        builder = PythonImageBuilder(config=config,
-                                     base_path=base_path,
-                                     relative_source_path='write_inputs',
-                                     tag=self.__IMAGE_NAME)
+        builder = PythonImageBuilder(
+            config=config, base_path=base_path, relative_source_path='write_inputs', tag=self.__IMAGE_NAME
+        )
 
         build_out = str(builder.build())
 
@@ -90,18 +88,12 @@ class TestPythonImageBuilder(TestCase):
 
         cmds = ['/bin/bash', '-c', 'python write_inputs.py']
 
-        container_log = docker_client.containers.run(self.__IMAGE_NAME,
-                                                     cmds,
-                                                     volumes={
-                                                         self.temp_dir: {
-                                                             'bind': '/mnt/vol1',
-                                                             'mode': 'rw'
-                                                         }
-                                                     },
-                                                     environment={
-                                                         'NUM_FILES': 10,
-                                                         'NUM_SPLITS': 3
-                                                     })
+        container_log = docker_client.containers.run(
+            self.__IMAGE_NAME,
+            cmds,
+            volumes={self.temp_dir: {'bind': '/mnt/vol1', 'mode': 'rw'}},
+            environment={'NUM_FILES': 10, 'NUM_SPLITS': 3},
+        )
 
         docker_client.close()
 
@@ -120,7 +112,8 @@ class TestPythonImageBuilder(TestCase):
             "Writing input file /mnt/vol1/inputs/2/input8.json\\n"
             "Writing input file /mnt/vol1/inputs/0/input9.json\\n"
             "'",
-            str(container_log))
+            str(container_log),
+        )
 
     def __create_conf(self, task_id):
         return {
diff --git a/tests/runners/airflow/build/spark/test_spark_image_builder.py b/tests/runners/airflow/build/spark/test_spark_image_builder.py
index 70148b4..8637303 100644
--- a/tests/runners/airflow/build/spark/test_spark_image_builder.py
+++ b/tests/runners/airflow/build/spark/test_spark_image_builder.py
@@ -43,8 +43,7 @@ class TestSparkImageBuilder(TestCase):
 
     def test_build(self):
         build_out = self.__test_build()
-        self.assertTrue('RUN pip install -r requirements.txt' in build_out,
-                        'Incorrect pip command')
+        self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command')
 
         self.__test_image()
 
@@ -53,10 +52,9 @@ class TestSparkImageBuilder(TestCase):
 
         base_path = os.path.join(os.path.dirname(__file__), '../../../apps/test_spark_app')
 
-        builder = SparkImageBuilder(config=config,
-                                    base_path=base_path,
-                                    relative_source_path='wordcount',
-                                    tag=self.__IMAGE_NAME)
+        builder = SparkImageBuilder(
+            config=config, base_path=base_path, relative_source_path='wordcount', tag=self.__IMAGE_NAME
+        )
 
         build_out = str(builder.build())
 
@@ -68,14 +66,9 @@ class TestSparkImageBuilder(TestCase):
 
         cmds = ['spark-submit', 'wordcount.py', 'words.txt', '/mnt/vol1/outputs']
 
-        container_log = docker_client.containers.run(self.__IMAGE_NAME,
-                                                     cmds,
-                                                     volumes={
-                                                         self.temp_dir: {
-                                                             'bind': '/mnt/vol1',
-                                                             'mode': 'rw'
-                                                         }
-                                                     })
+        container_log = docker_client.containers.run(
+            self.__IMAGE_NAME, cmds, volumes={self.temp_dir: {'bind': '/mnt/vol1', 'mode': 'rw'}}
+        )
 
         docker_client.close()
 
@@ -89,7 +82,8 @@ class TestSparkImageBuilder(TestCase):
             "spark: 1\\n"
             "task: 1\\n"
             "writing the results to /mnt/vol1/outputs\\n'",
-            str(container_log))
+            str(container_log),
+        )
 
     def __create_conf(self, task_id):
         return {
diff --git a/tests/runners/airflow/build/test_liminal_apps_builder.py b/tests/runners/airflow/build/test_liminal_apps_builder.py
index 2a9c467..e9f3a82 100644
--- a/tests/runners/airflow/build/test_liminal_apps_builder.py
+++ b/tests/runners/airflow/build/test_liminal_apps_builder.py
@@ -24,11 +24,9 @@ import docker
 
 from liminal.build import liminal_apps_builder
 
+
 class TestLiminalAppsBuilder(TestCase):
-    __image_names = [
-        'my_python_task_img',
-        'my_parallelized_python_task_img'
-    ]
+    __image_names = ['my_python_task_img', 'my_parallelized_python_task_img']
 
     def setUp(self) -> None:
         self.docker_client = docker.client.from_env()
@@ -44,8 +42,7 @@ class TestLiminalAppsBuilder(TestCase):
                 self.docker_client.images.remove(image=image_name, force=True)
 
     def test_build_liminal(self):
-        liminal_apps_builder.build_liminal_apps(
-            os.path.join(os.path.dirname(__file__), '../liminal'))
+        liminal_apps_builder.build_liminal_apps(os.path.join(os.path.dirname(__file__), '../liminal'))
 
         for image in self.__image_names:
             self.docker_client.images.get(image)
diff --git a/tests/runners/airflow/executors/test_emr.py b/tests/runners/airflow/executors/test_emr.py
index 19c7b99..e530783 100644
--- a/tests/runners/airflow/executors/test_emr.py
+++ b/tests/runners/airflow/executors/test_emr.py
@@ -28,8 +28,7 @@ from moto import mock_emr
 
 from liminal.runners.airflow import DummyDag
 from liminal.runners.airflow.executors.emr import EMRExecutor
-from liminal.runners.airflow.operators.operator_with_variable_resolving import \
-    OperatorWithVariableResolving
+from liminal.runners.airflow.operators.operator_with_variable_resolving import OperatorWithVariableResolving
 from liminal.runners.airflow.tasks import hadoop
 from tests.util import dag_test_utils
 
@@ -53,7 +52,7 @@ class TestEMRExecutorTask(TestCase):
             LogUri="s3://liminal/log",
             Name="test-emr-cluster",
             ServiceRole="EMR_DefaultRole",
-            VisibleToAllUsers=True
+            VisibleToAllUsers=True,
         )
 
         self.client = boto3.client("emr", region_name="us-east-1")
@@ -70,14 +69,10 @@ class TestEMRExecutorTask(TestCase):
             'cluster_name': self.executor_name,
             'aws_conn_id': 'us-east-1',
             'type': 'emr',
-            'properties': {
-                'ActionOnFailure': 'CONTINUE'
-            }
-
+            'properties': {'ActionOnFailure': 'CONTINUE'},
         }
         self.hadoop_task = MagicMock(spec=hadoop.HadoopTask)
-        self.hadoop_task.get_runnable_command.return_value = ['spark-submit', 'test', 'params',
-                                                              '--param']
+        self.hadoop_task.get_runnable_command.return_value = ['spark-submit', 'test', 'params', '--param']
         self.hadoop_task.task_id = 'spark-task'
         self.hadoop_task.dag = self.dag
         self.hadoop_task.trigger_rule = 'all_done'
@@ -85,11 +80,7 @@ class TestEMRExecutorTask(TestCase):
         self.hadoop_task.task_config = {}
         self.hadoop_task.variables = {}
 
-        self.emr = EMRExecutor(
-            self.executor_name,
-            liminal_config={},
-            executor_config=executor_config
-        )
+        self.emr = EMRExecutor(self.executor_name, liminal_config={}, executor_config=executor_config)
 
     def test_apply_task_to_dag(self):
         self.emr.apply_task_to_dag(task=self.hadoop_task)
@@ -104,10 +95,7 @@ class TestEMRExecutorTask(TestCase):
     @mock.patch.object(EmrHook, 'get_conn')
     def test_add_step(self, mock_emr_hook_get_conn):
         aws_region = 'us-east-1'
-        mock_emr_hook_get_conn.return_value = boto3.client(
-            'emr',
-            region_name=aws_region
-        )
+        mock_emr_hook_get_conn.return_value = boto3.client('emr', region_name=aws_region)
 
         self.emr.apply_task_to_dag(task=self.hadoop_task)
 
@@ -120,16 +108,12 @@ class TestEMRExecutorTask(TestCase):
 
         step_id = emr_add_step_task.execute(self.dag.context)[0]
 
-        desc_step = self.client.describe_step(
-            ClusterId=self.cluster_id,
-            StepId=step_id
-        )['Step']
+        desc_step = self.client.describe_step(ClusterId=self.cluster_id, StepId=step_id)['Step']
 
-        self.assertEqual(desc_step['Config'],
-                         {'Jar': 'command-runner.jar',
-                          'Properties': {},
-                          'Args': ['spark-submit', 'test', 'params', '--param']
-                          })
+        self.assertEqual(
+            desc_step['Config'],
+            {'Jar': 'command-runner.jar', 'Properties': {}, 'Args': ['spark-submit', 'test', 'params', '--param']},
+        )
 
         self.assertEqual(desc_step['Name'], 'spark-task')
 
diff --git a/tests/runners/airflow/liminal/write_outputs/write_outputs.py b/tests/runners/airflow/liminal/write_outputs/write_outputs.py
index fb75176..fbd6076 100644
--- a/tests/runners/airflow/liminal/write_outputs/write_outputs.py
+++ b/tests/runners/airflow/liminal/write_outputs/write_outputs.py
@@ -32,11 +32,9 @@ for directory in os.listdir(inputs_dir):
     logging.info(f'Running write_outputs for split id {directory}')
     inputs_dir_file = os.path.join(inputs_dir, directory)
     for filename in os.listdir(inputs_dir_file):
-        with open(os.path.join(inputs_dir_file, filename)) as infile, \
-                open(os.path.join(
-                    outputs_dir,
-                    filename.replace('input', 'output').replace('.json', '.txt')
-                ), 'w') as outfile:
+        with open(os.path.join(inputs_dir_file, filename)) as infile, open(
+            os.path.join(outputs_dir, filename.replace('input', 'output').replace('.json', '.txt')), 'w'
+        ) as outfile:
             logging.info(f'Writing output file: {outfile.name}')
             data = json.loads(infile.read())
             outfile.write(data['mykey'])
diff --git a/tests/runners/airflow/operators/test_operator_with_variable_resolving.py b/tests/runners/airflow/operators/test_operator_with_variable_resolving.py
index 53a875d..1207feb 100644
--- a/tests/runners/airflow/operators/test_operator_with_variable_resolving.py
+++ b/tests/runners/airflow/operators/test_operator_with_variable_resolving.py
@@ -22,120 +22,122 @@ from unittest.mock import patch, MagicMock
 
 from airflow.models import BaseOperator
 
-from liminal.runners.airflow.operators.operator_with_variable_resolving import \
-    OperatorWithVariableResolving
+from liminal.runners.airflow.operators.operator_with_variable_resolving import OperatorWithVariableResolving
 
 
 class TestKubernetesPodOperatorWithAutoImage(TestCase):
-    @patch.dict(os.environ,
-                {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True', 'nested': 'value1'})
+    @patch.dict(os.environ, {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True', 'nested': 'value1'})
     def test_value_from_environment_and_liminal_config(self):
         variables = {'my-image': 'my-image', 'something-value1': 'stg'}
         operator = TestOperator(task_id='abc', field='{{env}}', expected='{{my-image}}')
-        ret_val = OperatorWithVariableResolving(dag=None,
-                                                operator=operator,
-                                                task_id='my_task',
-                                                task_config={},
-                                                variables=variables).execute({})
+        ret_val = OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables=variables
+        ).execute({})
 
         self.assertEqual(operator.field, 'myenv')
         self.assertEqual(ret_val, 'my-image')
 
-    @patch.dict(os.environ,
-                {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True', 'nested': 'value1'})
+    @patch.dict(os.environ, {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True', 'nested': 'value1'})
     def test_value_from_task_variables(self):
         variables = {'my-image': 'my-image', 'my_dict_var': {'env': 'myenv2'}}
         operator = TestOperator(task_id='abc', field='{{env}}', expected='{{my-image}}')
-        ret_val = OperatorWithVariableResolving(dag=None,
-                                                operator=operator,
-                                                task_id='my_task',
-                                                task_config={'variables': 'my_dict_var'},
-                                                variables=variables).execute({})
+        ret_val = OperatorWithVariableResolving(
+            dag=None,
+            operator=operator,
+            task_id='my_task',
+            task_config={'variables': 'my_dict_var'},
+            variables=variables,
+        ).execute({})
 
         self.assertEqual(operator.field, 'myenv2')
         self.assertEqual(ret_val, 'my-image')
 
-    @patch.dict(os.environ,
-                {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True', 'nested': 'value1'})
+    @patch.dict(os.environ, {'env': 'myenv', 'LIMINAL_STAND_ALONE_MODE': 'True', 'nested': 'value1'})
     def test_value_from_inline_task_variables(self):
         variables = {'my-image': 'my-image'}
         operator = TestOperator(task_id='abc', field='{{env}}', expected='{{my-image}}')
-        ret_val = OperatorWithVariableResolving(dag=None,
-                                                operator=operator,
-                                                task_id='my_task',
-                                                task_config={'variables': {'env': 'myenv2'}},
-                                                variables=variables).execute({})
+        ret_val = OperatorWithVariableResolving(
+            dag=None,
+            operator=operator,
+            task_id='my_task',
+            task_config={'variables': {'env': 'myenv2'}},
+            variables=variables,
+        ).execute({})
 
         self.assertEqual(operator.field, 'myenv2')
         self.assertEqual(ret_val, 'my-image')
 
     @patch.dict(os.environ, {'env': 'staging', 'LIMINAL_STAND_ALONE_MODE': 'True'})
     def test_nested_variables(self):
-        variables = {'nested-the-s3-location': 'good', 'my-image': 'my-image',
-                     'something-value1': 'stg', 'env_val-staging': 'the-s3-location',
-                     'nested': 'value1'}
-        operator = TestOperator(task_id='abc', field='something-{{nested-{{env_val-{{env}}}}}}',
-                                expected='{{my-image}}')
-
-        OperatorWithVariableResolving(dag=None,
-                                      operator=operator,
-                                      task_id='my_task',
-                                      task_config={},
-                                      variables=variables).execute({})
+        variables = {
+            'nested-the-s3-location': 'good',
+            'my-image': 'my-image',
+            'something-value1': 'stg',
+            'env_val-staging': 'the-s3-location',
+            'nested': 'value1',
+        }
+        operator = TestOperator(
+            task_id='abc', field='something-{{nested-{{env_val-{{env}}}}}}', expected='{{my-image}}'
+        )
+
+        OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables=variables
+        ).execute({})
 
         self.assertEqual(operator.field, 'something-good')
 
     @patch.dict(os.environ, {'env': 'staging', 'LIMINAL_STAND_ALONE_MODE': 'True'})
     def test_dag_run_conf_parameters(self):
         dag_run = MagicMock()
-        dag_run.conf = {'nested-the-s3-location': 'good', 'my-image': 'my-image',
-                        'something-value1': 'stg', 'env_val-staging': 'the-s3-location',
-                        'nested': 'value1'}
-        operator = TestOperator(task_id='abc', field='something-{{nested-{{env_val-{{env}}}}}}',
-                                expected='{{my-image}}')
-        OperatorWithVariableResolving(dag=None,
-                                      operator=operator,
-                                      task_id='my_task',
-                                      task_config={},
-                                      variables={}).execute({'dag_run': dag_run})
+        dag_run.conf = {
+            'nested-the-s3-location': 'good',
+            'my-image': 'my-image',
+            'something-value1': 'stg',
+            'env_val-staging': 'the-s3-location',
+            'nested': 'value1',
+        }
+        operator = TestOperator(
+            task_id='abc', field='something-{{nested-{{env_val-{{env}}}}}}', expected='{{my-image}}'
+        )
+        OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables={}
+        ).execute({'dag_run': dag_run})
 
         self.assertEqual(operator.field, 'something-good')
 
     @patch.dict(os.environ, {'env': 'staging', 'LIMINAL_STAND_ALONE_MODE': 'True'})
     def test_duplicated_params(self):
         dag_run = MagicMock()
-        dag_run.conf = {'nested-the-s3-location': 'good', 'my-image': 'my-image',
-                        'something-value1': 'stg', 'env_val-staging': 'the-s3-location',
-                        'nested': 'value1'}
-        operator = TestOperator(task_id='abc', field='something-{{nested-{{env_val-{{env}}}}}}-'
-                                                     '{{nested-{{env_val-{{env}}}}}}',
-                                expected='{{my-image}}')
-        OperatorWithVariableResolving(dag=None,
-                                      operator=operator,
-                                      task_id='my_task',
-                                      task_config={},
-                                      variables={}).execute({'dag_run': dag_run})
+        dag_run.conf = {
+            'nested-the-s3-location': 'good',
+            'my-image': 'my-image',
+            'something-value1': 'stg',
+            'env_val-staging': 'the-s3-location',
+            'nested': 'value1',
+        }
+        operator = TestOperator(
+            task_id='abc',
+            field='something-{{nested-{{env_val-{{env}}}}}}-' '{{nested-{{env_val-{{env}}}}}}',
+            expected='{{my-image}}',
+        )
+        OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables={}
+        ).execute({'dag_run': dag_run})
 
         self.assertEqual(operator.field, 'something-good-good')
 
-        operator = TestOperator(task_id='abc', field='something-{{env}} {{env}}',
-                                expected='{{my-image}}')
-        OperatorWithVariableResolving(dag=None,
-                                      operator=operator,
-                                      task_id='my_task',
-                                      task_config={},
-                                      variables={}).execute({'dag_run': dag_run})
+        operator = TestOperator(task_id='abc', field='something-{{env}} {{env}}', expected='{{my-image}}')
+        OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables={}
+        ).execute({'dag_run': dag_run})
 
         self.assertEqual(operator.field, 'something-staging staging')
 
         dag_run.conf = {'var1': '{{var2}}', 'var2': 'bla', 'my-image': 'my-image'}
-        operator = TestOperator(task_id='abc', field='{{ var1 }}',
-                                expected='{{my-image}}')
-        OperatorWithVariableResolving(dag=None,
-                                      operator=operator,
-                                      task_id='my_task',
-                                      task_config={},
-                                      variables={}).execute({'dag_run': dag_run})
+        operator = TestOperator(task_id='abc', field='{{ var1 }}', expected='{{my-image}}')
+        OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables={}
+        ).execute({'dag_run': dag_run})
 
         self.assertEqual(operator.field, 'bla')
 
@@ -143,19 +145,15 @@ class TestKubernetesPodOperatorWithAutoImage(TestCase):
     def test_non_existing_param(self):
         dag_run = MagicMock()
         dag_run.conf = {'var1': '{{var2}}', 'var2': 'bla'}
-        operator = TestOperator(task_id='abc', field='{{myimage}}',
-                                expected='{{myimage}}')
-        OperatorWithVariableResolving(dag=None,
-                                      operator=operator,
-                                      task_id='my_task',
-                                      task_config={},
-                                      variables={}).execute({'dag_run': dag_run})
+        operator = TestOperator(task_id='abc', field='{{myimage}}', expected='{{myimage}}')
+        OperatorWithVariableResolving(
+            dag=None, operator=operator, task_id='my_task', task_config={}, variables={}
+        ).execute({'dag_run': dag_run})
 
         self.assertEqual(operator.field, '')
 
 
 class BaseTestOperator(BaseOperator):
-
     def execute(self, context):
         raise NotImplementedError()
 
@@ -166,7 +164,6 @@ class BaseTestOperator(BaseOperator):
 
 
 class TestOperator(BaseTestOperator):
-
     def __init__(self, *args, **kwargs):
         super(TestOperator, self).__init__(*args, **kwargs)
 
diff --git a/tests/runners/airflow/tasks/test_create_cloudformation_stack.py b/tests/runners/airflow/tasks/test_create_cloudformation_stack.py
index e57dad4..63557d9 100644
--- a/tests/runners/airflow/tasks/test_create_cloudformation_stack.py
+++ b/tests/runners/airflow/tasks/test_create_cloudformation_stack.py
@@ -24,8 +24,11 @@ from airflow.operators.python import BranchPythonOperator
 from moto import mock_cloudformation
 
 from liminal.runners.airflow.executors import airflow
-from liminal.runners.airflow.operators.cloudformation import CloudFormationCreateStackOperator, \
-    CloudFormationCreateStackSensor, CloudFormationHook
+from liminal.runners.airflow.operators.cloudformation import (
+    CloudFormationCreateStackOperator,
+    CloudFormationCreateStackSensor,
+    CloudFormationHook,
+)
 from liminal.runners.airflow.operators.operator_with_variable_resolving import OperatorWithVariableResolving
 from liminal.runners.airflow.tasks.create_cloudformation_stack import CreateCloudFormationStackTask
 from tests.util import dag_test_utils
@@ -43,7 +46,7 @@ class TestCreateCloudFormationStackTask(TestCase):
         self.dag.context = {
             'ti': self.dag,
             'ts': datetime.now().timestamp(),
-            'execution_date': datetime.now().timestamp()
+            'execution_date': datetime.now().timestamp(),
         }
         self.dag.get_dagrun = MagicMock()
 
@@ -66,24 +69,22 @@ class TestCreateCloudFormationStackTask(TestCase):
                     'CoreServerCount': '1',
                     'EmrApplicationRelease': '5.28',
                     'InstanceTypeMaster': 'm5.xlarge',
-                    'InstanceTypeCore': 'm5.xlarge'
-                }
-            }
+                    'InstanceTypeCore': 'm5.xlarge',
+                },
+            },
         }
 
-        self.create_cloudformation_task = \
-            CreateCloudFormationStackTask(
-                self.config['task'],
-                self.dag,
-                [],
-                trigger_rule='all_success',
-                liminal_config={},
-                pipeline_config={},
-                task_config=self.config
-            )
+        self.create_cloudformation_task = CreateCloudFormationStackTask(
+            self.config['task'],
+            self.dag,
+            [],
+            trigger_rule='all_success',
+            liminal_config={},
+            pipeline_config={},
+            task_config=self.config,
+        )
 
-        airflow.AirflowExecutor("airflow-executor", {}, {}).apply_task_to_dag(
-            task=self.create_cloudformation_task)
+        airflow.AirflowExecutor("airflow-executor", {}, {}).apply_task_to_dag(task=self.create_cloudformation_task)
 
     def test_apply_task_to_dag(self):
         self.assertEqual(len(self.dag.tasks), 4)
@@ -107,7 +108,8 @@ class TestCreateCloudFormationStackTask(TestCase):
             print(is_cloudformation_exists)
             self.assertEqual(
                 is_cloudformation_exists.python_callable(stack_name=self.cluster_name),
-                'create-cloudformation-create_emr')
+                'create-cloudformation-create_emr',
+            )
 
     def test_cloudformation_exist_and_running(self):
         is_cloudformation_exists = self.dag.tasks[0].operator_delegate
@@ -115,19 +117,13 @@ class TestCreateCloudFormationStackTask(TestCase):
         for status in ['CREATE_COMPLETE', 'DELETE_FAILED']:
             with mock.patch.object(CloudFormationHook, 'get_conn') as mock_conn:
                 mock_cloudformation_conn = MagicMock()
-                mock_cloudformation_conn.describe_stacks.return_value = {
-                    'Stacks': [
-                        {
-                            'StackStatus': status
-                        }
-                    ]
-                }
+                mock_cloudformation_conn.describe_stacks.return_value = {'Stacks': [{'StackStatus': status}]}
 
                 mock_conn.return_value = mock_cloudformation_conn
 
                 self.assertEqual(
-                    is_cloudformation_exists.python_callable(stack_name=self.cluster_name),
-                    'creation-end-create_emr')
+                    is_cloudformation_exists.python_callable(stack_name=self.cluster_name), 'creation-end-create_emr'
+                )
 
     def test_cloudformation_exists_and_not_running(self):
         is_cloudformation_exists = self.dag.tasks[0].operator_delegate
@@ -135,19 +131,14 @@ class TestCreateCloudFormationStackTask(TestCase):
         for status in ['DELETED']:
             with mock.patch.object(CloudFormationHook, 'get_conn') as mock_conn:
                 mock_cloudformation_conn = MagicMock()
-                mock_cloudformation_conn.describe_stacks.return_value = {
-                    'Stacks': [
-                        {
-                            'StackStatus': status
-                        }
-                    ]
-                }
+                mock_cloudformation_conn.describe_stacks.return_value = {'Stacks': [{'StackStatus': status}]}
 
                 mock_conn.return_value = mock_cloudformation_conn
 
                 self.assertEqual(
                     is_cloudformation_exists.python_callable(stack_name=self.cluster_name),
-                    'create-cloudformation-create_emr')
+                    'create-cloudformation-create_emr',
+                )
 
     def test_cloudformation_create_stack_operator_task(self):
         create_cloudformation_stack = self.dag.tasks[1]
diff --git a/tests/runners/airflow/tasks/test_delete_cloudformation_stack.py b/tests/runners/airflow/tasks/test_delete_cloudformation_stack.py
index dbb2656..7594ff6 100644
--- a/tests/runners/airflow/tasks/test_delete_cloudformation_stack.py
+++ b/tests/runners/airflow/tasks/test_delete_cloudformation_stack.py
@@ -25,8 +25,10 @@ from airflow.operators.python_operator import BranchPythonOperator
 from airflow.utils.trigger_rule import TriggerRule
 from moto import mock_cloudformation
 
-from liminal.runners.airflow.operators.cloudformation import CloudFormationDeleteStackOperator, \
-    CloudFormationDeleteStackSensor
+from liminal.runners.airflow.operators.cloudformation import (
+    CloudFormationDeleteStackOperator,
+    CloudFormationDeleteStackSensor,
+)
 from liminal.runners.airflow.operators.operator_with_variable_resolving import OperatorWithVariableResolving
 from liminal.runners.airflow.tasks.delete_cloudformation_stack import DeleteCloudFormationStackTask
 from tests.util import dag_test_utils
@@ -44,24 +46,21 @@ class TestDeleteCloudFormationStackTask(TestCase):
         self.dag.context = {
             'ti': self.dag,
             'ts': datetime.now().timestamp(),
-            'execution_date': datetime.now().timestamp()
+            'execution_date': datetime.now().timestamp(),
         }
         self.dag.get_dagrun = MagicMock()
 
         self.cluster_name = "liminal-cluster-for-tests"
 
-        self.delete_cloudformation_task = \
-            DeleteCloudFormationStackTask(
-                'delete-emr',
-                self.dag,
-                [],
-                trigger_rule='all_success',
-                liminal_config={},
-                pipeline_config={},
-                task_config={
-                    'stack_name': self.cluster_name
-                }
-            )
+        self.delete_cloudformation_task = DeleteCloudFormationStackTask(
+            'delete-emr',
+            self.dag,
+            [],
+            trigger_rule='all_success',
+            liminal_config={},
+            pipeline_config={},
+            task_config={'stack_name': self.cluster_name},
+        )
 
         self.delete_cloudformation_task.apply_task_to_dag()
 
@@ -82,10 +81,8 @@ class TestDeleteCloudFormationStackTask(TestCase):
         check_dags_queued_task = self.dag.tasks[0]
         self.assertEqual(check_dags_queued_task.trigger_rule, TriggerRule.ALL_DONE)
 
-        self.assertEqual(check_dags_queued_task.python_callable(),
-                         f'delete-end-delete-emr')
+        self.assertEqual(check_dags_queued_task.python_callable(), f'delete-end-delete-emr')
 
         self.dag.get_num_active_runs.return_value = 1
 
-        self.assertEqual(check_dags_queued_task.python_callable(),
-                         f'delete-cloudformation-delete-emr')
+        self.assertEqual(check_dags_queued_task.python_callable(), f'delete-cloudformation-delete-emr')
diff --git a/tests/runners/airflow/tasks/test_job_end.py b/tests/runners/airflow/tasks/test_job_end.py
index 4e1148c..b6a4884 100644
--- a/tests/runners/airflow/tasks/test_job_end.py
+++ b/tests/runners/airflow/tasks/test_job_end.py
@@ -26,7 +26,6 @@ from tests.util import dag_test_utils
 
 # noinspection DuplicatedCode
 class TestJobEndTask(TestCase):
-
     def test_apply_task_to_dag(self):
         dag = dag_test_utils.create_dag()
 
@@ -37,7 +36,7 @@ class TestJobEndTask(TestCase):
             task_config={},
             parent=None,
             trigger_rule='all_done',
-            liminal_config={'metrics': {'namespace': 'EndJobNameSpace', 'backends': ['cloudwatch']}}
+            liminal_config={'metrics': {'namespace': 'EndJobNameSpace', 'backends': ['cloudwatch']}},
         )
         airflow.AirflowExecutor("airflow-executor", {}, {}).apply_task_to_dag(task=task0)
 
@@ -53,10 +52,15 @@ class TestJobEndTask(TestCase):
         conf = {'pipeline': 'my_pipeline'}
         dag = dag_test_utils.create_dag()
 
-        task0 = job_end.JobEndTask(task_id="job_end", dag=dag,
-                                   pipeline_config={'pipeline': 'my_end_pipeline'},
-                                   liminal_config=conf, parent=None,
-                                   trigger_rule='all_done', task_config={})
+        task0 = job_end.JobEndTask(
+            task_id="job_end",
+            dag=dag,
+            pipeline_config={'pipeline': 'my_end_pipeline'},
+            liminal_config=conf,
+            parent=None,
+            trigger_rule='all_done',
+            task_config={},
+        )
 
         airflow.AirflowExecutor("airflow-executor", {}, {}).apply_task_to_dag(task=task0)
 
@@ -70,13 +74,15 @@ class TestJobEndTask(TestCase):
     def test_apply_task_to_dag_with_partial_configuration(self):
         dag = dag_test_utils.create_dag()
 
-        task0 = job_end.JobEndTask(task_id="job_enc",
-                                   dag=dag,
-                                   liminal_config={'metrics': {'namespace': 'EndJobNameSpace'}},
-                                   pipeline_config={'pipeline': 'my_end_pipeline'},
-                                   task_config={},
-                                   parent=None,
-                                   trigger_rule='all_done')
+        task0 = job_end.JobEndTask(
+            task_id="job_enc",
+            dag=dag,
+            liminal_config={'metrics': {'namespace': 'EndJobNameSpace'}},
+            pipeline_config={'pipeline': 'my_end_pipeline'},
+            task_config={},
+            parent=None,
+            trigger_rule='all_done',
+        )
         task0.apply_task_to_dag()
 
         self.assertEqual(len(dag.tasks), 1)
diff --git a/tests/runners/airflow/tasks/test_job_start.py b/tests/runners/airflow/tasks/test_job_start.py
index 97faaa0..4175fab 100644
--- a/tests/runners/airflow/tasks/test_job_start.py
+++ b/tests/runners/airflow/tasks/test_job_start.py
@@ -26,19 +26,17 @@ from tests.util import dag_test_utils
 
 # noinspection DuplicatedCode
 class TestJobStartTask(TestCase):
-
     def test_apply_task_to_dag(self):
         dag = dag_test_utils.create_dag()
 
         task0 = job_start.JobStartTask(
             task_id="start_task",
             dag=dag,
-            liminal_config={
-                'metrics': {'namespace': 'StartJobNameSpace', 'backends': ['cloudwatch']}},
+            liminal_config={'metrics': {'namespace': 'StartJobNameSpace', 'backends': ['cloudwatch']}},
             pipeline_config={'pipeline': 'my_start_pipeline'},
             task_config={},
             parent=None,
-            trigger_rule='all_success'
+            trigger_rule='all_success',
         )
         task0.apply_task_to_dag()
 
@@ -62,7 +60,8 @@ class TestJobStartTask(TestCase):
             task_config={},
             pipeline_config={'pipeline': 'my_end_pipeline'},
             parent=None,
-            trigger_rule='all_success')
+            trigger_rule='all_success',
+        )
         airflow.AirflowExecutor("airflow-executor", {}, {}).apply_task_to_dag(task=task0)
 
         self.assertEqual(len(dag.tasks), 1)
@@ -75,14 +74,15 @@ class TestJobStartTask(TestCase):
     def test_apply_task_to_dag_with_partial_configuration(self):
         dag = dag_test_utils.create_dag()
 
-        task0 = job_start.JobStartTask(task_id="start_task",
-                                       dag=dag,
-                                       liminal_config={
-                                           'metrics': {'namespace': 'StartJobNameSpace'}},
-                                       pipeline_config={'pipeline': 'my_start_pipeline'},
-                                       task_config={},
-                                       parent=None,
-                                       trigger_rule='all_success', )
+        task0 = job_start.JobStartTask(
+            task_id="start_task",
+            dag=dag,
+            liminal_config={'metrics': {'namespace': 'StartJobNameSpace'}},
+            pipeline_config={'pipeline': 'my_start_pipeline'},
+            task_config={},
+            parent=None,
+            trigger_rule='all_success',
+        )
         airflow.AirflowExecutor("airflow-executor", {}, {}).apply_task_to_dag(task=task0)
 
         self.assertEqual(len(dag.tasks), 1)
diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py
index 41f8379..0195dc6 100644
--- a/tests/runners/airflow/tasks/test_python.py
+++ b/tests/runners/airflow/tasks/test_python.py
@@ -40,49 +40,39 @@ class TestPythonTask(TestCase):
             'volumes': [
                 {
                     'volume': self._VOLUME_NAME,
-                    'local': {
-                        'path': self.temp_dir.replace(
-                            "/var/folders",
-                            "/private/var/folders"
-                        )
-                    }
+                    'local': {'path': self.temp_dir.replace("/var/folders", "/private/var/folders")},
                 }
             ]
         }
         volume_util.create_local_volumes(self.liminal_config, None)
 
-        liminal_apps_builder.build_liminal_apps(
-            os.path.join(os.path.dirname(__file__), '../liminal'))
+        liminal_apps_builder.build_liminal_apps(os.path.join(os.path.dirname(__file__), '../liminal'))
 
     def test_apply_task_to_dag(self):
         dag = dag_test_utils.create_dag()
 
-        task0 = self.__create_python_task(dag,
-                                          'my_input_task',
-                                          None,
-                                          'my_python_task_img',
-                                          'python -u write_inputs.py',
-                                          env_vars={
-                                              'NUM_FILES': 10,
-                                              'NUM_SPLITS': 3
-                                          })
+        task0 = self.__create_python_task(
+            dag,
+            'my_input_task',
+            None,
+            'my_python_task_img',
+            'python -u write_inputs.py',
+            env_vars={'NUM_FILES': 10, 'NUM_SPLITS': 3},
+        )
         executor = KubernetesPodExecutor(
-            task_id='k8s',
-            liminal_config=self.liminal_config,
-            executor_config={
-                'executor': 'k8s',
-                'name': 'mypod'
-            }
+            task_id='k8s', liminal_config=self.liminal_config, executor_config={'executor': 'k8s', 'name': 'mypod'}
         )
 
         executor.apply_task_to_dag(task=task0)
 
-        task1 = self.__create_python_task(dag,
-                                          'my_output_task',
-                                          dag.tasks[0],
-                                          'my_parallelized_python_task_img',
-                                          'python -u write_outputs.py',
-                                          executors=3)
+        task1 = self.__create_python_task(
+            dag,
+            'my_output_task',
+            dag.tasks[0],
+            'my_parallelized_python_task_img',
+            'python -u write_outputs.py',
+            executors=3,
+        )
         executor.apply_task_to_dag(task=task1)
 
         for task in dag.tasks:
@@ -98,45 +88,46 @@ class TestPythonTask(TestCase):
 
         self.assertListEqual(inputs_dir_contents, ['0', '1', '2'])
 
-        self.assertListEqual(sorted(os.listdir(os.path.join(inputs_dir, '0'))),
-                             ['input0.json', 'input3.json', 'input6.json', 'input9.json'])
+        self.assertListEqual(
+            sorted(os.listdir(os.path.join(inputs_dir, '0'))),
+            ['input0.json', 'input3.json', 'input6.json', 'input9.json'],
+        )
 
-        self.assertListEqual(sorted(os.listdir(os.path.join(inputs_dir, '1'))),
-                             ['input1.json', 'input4.json', 'input7.json'])
+        self.assertListEqual(
+            sorted(os.listdir(os.path.join(inputs_dir, '1'))), ['input1.json', 'input4.json', 'input7.json']
+        )
 
-        self.assertListEqual(sorted(os.listdir(os.path.join(inputs_dir, '2'))),
-                             ['input2.json', 'input5.json', 'input8.json'])
+        self.assertListEqual(
+            sorted(os.listdir(os.path.join(inputs_dir, '2'))), ['input2.json', 'input5.json', 'input8.json']
+        )
 
-        self.assertListEqual(sorted(os.listdir(outputs_dir)),
-                             ['output0.txt', 'output1.txt',
-                              'output2.txt', 'output3.txt',
-                              'output4.txt', 'output5.txt',
-                              'output6.txt', 'output7.txt',
-                              'output8.txt', 'output9.txt'])
+        self.assertListEqual(
+            sorted(os.listdir(outputs_dir)),
+            [
+                'output0.txt',
+                'output1.txt',
+                'output2.txt',
+                'output3.txt',
+                'output4.txt',
+                'output5.txt',
+                'output6.txt',
+                'output7.txt',
+                'output8.txt',
+                'output9.txt',
+            ],
+        )
 
         for filename in os.listdir(outputs_dir):
             with open(os.path.join(outputs_dir, filename)) as f:
                 expected_file_content = filename.replace('output', 'myval').replace('.txt', '')
                 self.assertEqual(f.read(), expected_file_content)
 
-    def __create_python_task(self,
-                             dag,
-                             task_id,
-                             parent,
-                             image,
-                             cmd,
-                             env_vars=None,
-                             executors=None):
+    def __create_python_task(self, dag, task_id, parent, image, cmd, env_vars=None, executors=None):
 
         self.liminal_config['volumes'] = [
             {
                 'volume': self._VOLUME_NAME,
-                'local': {
-                    'path': self.temp_dir.replace(
-                        "/var/folders",
-                        "/private/var/folders"
-                    )
-                }
+                'local': {'path': self.temp_dir.replace("/var/folders", "/private/var/folders")},
             }
         ]
 
@@ -151,13 +142,7 @@ class TestPythonTask(TestCase):
             'cmd': cmd,
             'image': image,
             'env_vars': env_vars if env_vars is not None else {},
-            'mounts': [
-                {
-                    'mount': 'mymount',
-                    'volume': self._VOLUME_NAME,
-                    'path': '/mnt/vol1'
-                }
-            ]
+            'mounts': [{'mount': 'mymount', 'volume': self._VOLUME_NAME, 'path': '/mnt/vol1'}],
         }
 
         if executors:
@@ -167,12 +152,11 @@ class TestPythonTask(TestCase):
             task_id=task_id,
             dag=dag,
             liminal_config=self.liminal_config,
-            pipeline_config={
-                'pipeline': 'my_pipeline'
-            },
+            pipeline_config={'pipeline': 'my_pipeline'},
             task_config=task_config,
             parent=parent,
-            trigger_rule='all_success')
+            trigger_rule='all_success',
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/runners/airflow/tasks/test_spark_task.py b/tests/runners/airflow/tasks/test_spark_task.py
index b0ae799..2d0b985 100644
--- a/tests/runners/airflow/tasks/test_spark_task.py
+++ b/tests/runners/airflow/tasks/test_spark_task.py
@@ -43,20 +43,14 @@ class TestSparkTask(TestCase):
             'volumes': [
                 {
                     'volume': self._VOLUME_NAME,
-                    'local': {
-                        'path': self.temp_dir.replace(
-                            "/var/folders",
-                            "/private/var/folders"
-                        )
-                    }
+                    'local': {'path': self.temp_dir.replace("/var/folders", "/private/var/folders")},
                 }
             ]
         }
         volume_util.create_local_volumes(self.liminal_config, None)
 
         # build spark image
-        liminal_apps_builder.build_liminal_apps(
-            os.path.join(os.path.dirname(__file__), '../../apps/test_spark_app'))
+        liminal_apps_builder.build_liminal_apps(os.path.join(os.path.dirname(__file__), '../../apps/test_spark_app'))
 
         outputs_dir = os.path.join(self.temp_dir, 'outputs')
 
@@ -66,13 +60,7 @@ class TestSparkTask(TestCase):
             'application_source': 'wordcount.py',
             'application_arguments': ['words.txt', '/mnt/vol1/outputs/'],
             'env_vars': {},
-            'mounts': [
-                {
-                    'mount': 'mymount',
-                    'volume': self._VOLUME_NAME,
-                    'path': '/mnt/vol1'
-                }
-            ]
+            'mounts': [{'mount': 'mymount', 'volume': self._VOLUME_NAME, 'path': '/mnt/vol1'}],
         }
 
         dag = dag_test_utils.create_dag()
@@ -81,20 +69,14 @@ class TestSparkTask(TestCase):
             task_id="my_spark_task",
             dag=dag,
             liminal_config=self.liminal_config,
-            pipeline_config={
-                'pipeline': 'my_pipeline'
-            },
+            pipeline_config={'pipeline': 'my_pipeline'},
             task_config=task_config,
             parent=None,
-            trigger_rule='all_success')
+            trigger_rule='all_success',
+        )
 
         executor = KubernetesPodExecutor(
-            task_id='k8s',
-            liminal_config=self.liminal_config,
-            executor_config={
-                'executor': 'k8s',
-                'name': 'mypod'
-            }
+            task_id='k8s', liminal_config=self.liminal_config, executor_config={'executor': 'k8s', 'name': 'mypod'}
         )
         executor.apply_task_to_dag(task=task1)
 
@@ -102,11 +84,13 @@ class TestSparkTask(TestCase):
             print(f'Executing task {task.task_id}')
             task.execute(DummyDag('my_dag', task.task_id).context)
 
-        expected_output = '{"word":"my","count":1}\n' \
-                          '{"word":"first","count":1}\n' \
-                          '{"word":"liminal","count":1}\n' \
-                          '{"word":"spark","count":1}\n' \
-                          '{"word":"task","count":1}\n'.split("\n")
+        expected_output = (
+            '{"word":"my","count":1}\n'
+            '{"word":"first","count":1}\n'
+            '{"word":"liminal","count":1}\n'
+            '{"word":"spark","count":1}\n'
+            '{"word":"task","count":1}\n'.split("\n")
+        )
 
         actual = ''
         for filename in os.listdir(outputs_dir):
@@ -124,28 +108,32 @@ class TestSparkTask(TestCase):
             'conf': {
                 'spark.driver.memory': '1g',
                 'spark.driver.maxResultSize': '1g',
-                'spark.yarn.executor.memoryOverhead': '500M'
+                'spark.yarn.executor.memoryOverhead': '500M',
             },
             'application_arguments': {
-                '--query': "select * from "
-                           "my_table where date_prt >= "
-                           "'{{yesterday_ds}}'",
-                '--output': 'mytable'
-            }
+                '--query': "select * from " "my_table where date_prt >= " "'{{yesterday_ds}}'",
+                '--output': 'mytable',
+            },
         }
 
-        expected = ['spark-submit',
-                    '--master',
-                    'yarn',
-                    '--class',
-                    'org.apache.liminal.MySparkApp',
-                    '--conf',
-                    'spark.driver.maxResultSize=1g', '--conf', 'spark.driver.memory=1g', '--conf',
-                    'spark.yarn.executor.memoryOverhead=500M', 'my_app.py',
-                    '--query',
-                    "select * from my_table where "
-                    "date_prt >= '{{yesterday_ds}}'",
-                    '--output', 'mytable']
+        expected = [
+            'spark-submit',
+            '--master',
+            'yarn',
+            '--class',
+            'org.apache.liminal.MySparkApp',
+            '--conf',
+            'spark.driver.maxResultSize=1g',
+            '--conf',
+            'spark.driver.memory=1g',
+            '--conf',
+            'spark.yarn.executor.memoryOverhead=500M',
+            'my_app.py',
+            '--query',
+            "select * from my_table where " "date_prt >= '{{yesterday_ds}}'",
+            '--output',
+            'mytable',
+        ]
 
         actual = SparkTask(
             'my_spark_task',
@@ -154,7 +142,7 @@ class TestSparkTask(TestCase):
             trigger_rule='all_success',
             liminal_config={},
             pipeline_config={'pipeline': 'pipeline'},
-            task_config=task_config
+            task_config=task_config,
         ).get_runnable_command()
 
         self.assertEqual(actual.sort(), expected.sort())
@@ -163,17 +151,19 @@ class TestSparkTask(TestCase):
         task_config = {
             'application_source': 'my_app.py',
             'application_arguments': {
-                '--query': "select * from my_table where"
-                           " date_prt >= '{{yesterday_ds}}'",
-                '--output': 'myoutputtable'
-            }
+                '--query': "select * from my_table where" " date_prt >= '{{yesterday_ds}}'",
+                '--output': 'myoutputtable',
+            },
         }
 
-        expected = ['spark-submit', 'my_app.py',
-                    '--query',
-                    "select * from my_table where "
-                    "date_prt >= '{{yesterday_ds}}'",
-                    '--output', 'myoutputtable']
+        expected = [
+            'spark-submit',
+            'my_app.py',
+            '--query',
+            "select * from my_table where " "date_prt >= '{{yesterday_ds}}'",
+            '--output',
+            'myoutputtable',
+        ]
 
         actual = SparkTask(
             'my_spark_task',
@@ -182,7 +172,7 @@ class TestSparkTask(TestCase):
             trigger_rule='all_success',
             liminal_config={},
             pipeline_config={'pipeline': 'pipeline'},
-            task_config=task_config
+            task_config=task_config,
         ).get_runnable_command()
 
         self.assertEqual(actual.sort(), expected.sort())
@@ -194,30 +184,30 @@ class TestSparkTask(TestCase):
             'conf': {
                 'spark.driver.memory': '1g',
                 'spark.driver.maxResultSize': '1g',
-                'spark.yarn.executor.memoryOverhead': '500M'
+                'spark.yarn.executor.memoryOverhead': '500M',
             },
             'application_arguments': {
-                '--query': "select * from my_table where "
-                           "date_prt >= '{{yesterday_ds}}'",
-                '--output': 'myoutputtable'
-            }
+                '--query': "select * from my_table where " "date_prt >= '{{yesterday_ds}}'",
+                '--output': 'myoutputtable',
+            },
         }
 
-        expected = ['spark-submit',
-                    '--class',
-                    'org.apache.liminal.MySparkApp',
-                    '--conf',
-                    'spark.driver.maxResultSize=1g',
-                    '--conf',
-                    'spark.driver.memory=1g',
-                    '--conf',
-                    'spark.yarn.executor.memoryOverhead=500M',
-                    'my_app.py',
-                    '--query',
-                    'select * from my_table where '
-                    "date_prt >= '{{yesterday_ds}}'",
-                    '--output',
-                    'myoutputtable']
+        expected = [
+            'spark-submit',
+            '--class',
+            'org.apache.liminal.MySparkApp',
+            '--conf',
+            'spark.driver.maxResultSize=1g',
+            '--conf',
+            'spark.driver.memory=1g',
+            '--conf',
+            'spark.yarn.executor.memoryOverhead=500M',
+            'my_app.py',
+            '--query',
+            'select * from my_table where ' "date_prt >= '{{yesterday_ds}}'",
+            '--output',
+            'myoutputtable',
+        ]
 
         actual = SparkTask(
             'my_spark_task',
@@ -226,7 +216,7 @@ class TestSparkTask(TestCase):
             trigger_rule='all_success',
             liminal_config={},
             pipeline_config={'pipeline': 'pipeline'},
-            task_config=task_config
+            task_config=task_config,
         ).get_runnable_command()
 
         self.assertEqual(actual.sort(), expected.sort())
diff --git a/tests/runners/apps/test_spark_app/wordcount/wordcount.py b/tests/runners/apps/test_spark_app/wordcount/wordcount.py
index b7537a3..d678cfe 100644
--- a/tests/runners/apps/test_spark_app/wordcount/wordcount.py
+++ b/tests/runners/apps/test_spark_app/wordcount/wordcount.py
@@ -27,16 +27,10 @@ if __name__ == "__main__":
         print("Usage: wordcount <file> output <outputlocation>", file=sys.stderr)
         sys.exit(-1)
 
-    spark = SparkSession \
-        .builder \
-        .appName("PythonWordCount") \
-        .getOrCreate()
+    spark = SparkSession.builder.appName("PythonWordCount").getOrCreate()
 
-    lines = spark.read.text(sys.argv[1]).rdd.filter(lambda x: not x[0].startswith('#')).map(
-        lambda r: r[0])
-    counts = lines.flatMap(lambda x: x.split(' ')) \
-        .map(lambda x: (x, 1)) \
-        .reduceByKey(add)
+    lines = spark.read.text(sys.argv[1]).rdd.filter(lambda x: not x[0].startswith('#')).map(lambda r: r[0])
+    counts = lines.flatMap(lambda x: x.split(' ')).map(lambda x: (x, 1)).reduceByKey(add)
     output = counts.collect()
     for (word, count) in output:
         print("%s: %i" % (word, count))
diff --git a/tests/test_licenses.py b/tests/test_licenses.py
index 6b4a3b4..b30194a 100644
--- a/tests/test_licenses.py
+++ b/tests/test_licenses.py
@@ -22,9 +22,7 @@ from unittest import TestCase
 
 from termcolor import colored
 
-EXCLUDED_EXTENSIONS = [
-    '.gif', '.png', '.pyc', 'LICENSE', 'DISCLAIMER', 'DISCLAIMER-WIP', 'NOTICE', '.whl'
-]
+EXCLUDED_EXTENSIONS = ['.gif', '.png', '.pyc', 'LICENSE', 'DISCLAIMER', 'DISCLAIMER-WIP', 'NOTICE', '.whl']
 EXCLUDED_DIRS = ['docs/build', 'build', 'dist', '.git', '.idea', 'venv', 'apache_liminal.egg-info']
 EXCLUDED_FILES = ['DISCLAIMER-WIP']
 
@@ -46,7 +44,9 @@ PYTHON_LICENSE_HEADER = """
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-""".strip().split("\n")
+""".strip().split(
+    "\n"
+)
 
 MD_LICENSE_HEADER = """
 <!--
@@ -67,7 +67,9 @@ KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 -->
-""".strip().split("\n")
+""".strip().split(
+    "\n"
+)
 
 RST_LICENSE_HEADER = """
 ..
@@ -90,7 +92,9 @@ RST_LICENSE_HEADER = """
    specific language governing permissions and limitations
    under the License.
 ..
-""".strip().split("\n")
+""".strip().split(
+    "\n"
+)
 
 BAT_LICENSE_HEADER = """
 REM
@@ -110,7 +114,9 @@ REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 REM KIND, either express or implied.  See the License for the
 REM specific language governing permissions and limitations
 REM under the License.
-""".strip().split("\n")
+""".strip().split(
+    "\n"
+)
 
 JSON_LICENSE_HEADER = """
   "": [
@@ -131,21 +137,23 @@ JSON_LICENSE_HEADER = """
     "specific language governing permissions and limitations",
     "under the License."
   ],
-""".strip().split("\n")
+""".strip().split(
+    "\n"
+)
 
 
 class TestLicenses(TestCase):
-
     def test_licenses(self):
         files = []
         base_dir = os.path.join(pathlib.Path(__file__).parent.parent.absolute())
         print(f'Checking licenses for files in {base_dir}')
         for r, d, f in os.walk(base_dir):
-            if not any(os.path.relpath(r, base_dir).startswith(excluded) for excluded in
-                       EXCLUDED_DIRS):
+            if not any(os.path.relpath(r, base_dir).startswith(excluded) for excluded in EXCLUDED_DIRS):
                 for file in f:
-                    if not any(os.path.basename(file).endswith(ext) for ext in EXCLUDED_EXTENSIONS) \
-                            and not os.path.basename(file) in EXCLUDED_FILES:
+                    if (
+                        not any(os.path.basename(file).endswith(ext) for ext in EXCLUDED_EXTENSIONS)
+                        and not os.path.basename(file) in EXCLUDED_FILES
+                    ):
                         files.append(os.path.join(r, file))
 
         output = ''
diff --git a/tests/util/dag_test_utils.py b/tests/util/dag_test_utils.py
index b1fbcab..5a173f0 100644
--- a/tests/util/dag_test_utils.py
+++ b/tests/util/dag_test_utils.py
@@ -27,7 +27,4 @@ def create_dag():
     Test util to create a basic DAG for testing.
     """
 
-    return DAG(
-        dag_id='test_dag',
-        default_args={'start_date': datetime(1970, 1, 1)}
-    )
+    return DAG(dag_id='test_dag', default_args={'start_date': datetime(1970, 1, 1)})
diff --git a/tests/util/test_class_utils.py b/tests/util/test_class_utils.py
index bdfacfa..82d3b3f 100644
--- a/tests/util/test_class_utils.py
+++ b/tests/util/test_class_utils.py
@@ -32,7 +32,7 @@ class Test(TestCase):
             'test_clazz_child_1': B,
             'test_clazz_child_2': C,
             'test_clazz_leaf_1': D,
-            'test_clazz_leaf_2': E
+            'test_clazz_leaf_2': E,
         }
         self.hierarchy_check(A, expected_set)