You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marvin.apache.org by we...@apache.org on 2021/10/23 13:13:16 UTC
[incubator-marvin] 01/04: Quick fixes at CLI and Airflow/Kubernetes
templates
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit f5f96396fd6cb7ec64ef3e96ae85fb80153eec28
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Oct 20 20:36:23 2021 -0300
Quick fixes at CLI and Airflow/Kubernetes templates
---
marvin-cli/MANIFEST.in | 2 +
marvin-cli/Makefile | 0
marvin-cli/README.md | 0
marvin-cli/bin/marvin | 59 ++--
marvin-cli/bin/marvin-api | 50 +++
marvin-cli/bin/marvin_complete | 0
marvin-cli/marvin_cli/VERSION | 0
marvin-cli/marvin_cli/__init__.py | 0
marvin-cli/marvin_cli/communication/__init__.py | 0
.../marvin_cli/communication/remote_calls.py | 18 +-
.../marvin_cli/communication/stubs/__init__.py | 0
.../marvin_cli/communication/stubs/daemon_pb2.py | 0
.../communication/stubs/daemon_pb2_grpc.py | 0
marvin-cli/marvin_cli/management/__init__.py | 0
.../management/airflow_template/cookiecutter.json | 8 +
.../{{cookiecutter.dag_name}}.py | 53 +++
marvin-cli/marvin_cli/management/edit.py | 6 -
marvin-cli/marvin_cli/management/engine.py | 379 ++++++++++++++++-----
marvin-cli/marvin_cli/management/generate.py | 56 +++
.../kubernetes_template/cookiecutter.json | 9 +
.../{{cookiecutter.engine_name}}/deploy.sh | 6 +
.../{{cookiecutter.engine_name}}_deployment.yaml | 21 ++
.../{{cookiecutter.engine_name}}_service.yaml | 11 +
marvin-cli/marvin_cli/management/notebook.py | 34 +-
.../management/template/cookiecutter.json | 0
.../{{cookiecutter.project_name}}/.bumpversion.cfg | 0
.../{{cookiecutter.project_name}}/.coveragerc | 0
.../{{cookiecutter.project_name}}/.gitignore | 0
.../{{cookiecutter.project_name}}/CHANGES.md | 0
.../template/{{cookiecutter.project_name}}/INSTALL | 0
.../template/{{cookiecutter.project_name}}/LICENSE | 0
.../{{cookiecutter.project_name}}/MANIFEST.in | 3 +-
.../{{cookiecutter.project_name}}/Makefile | 0
.../{{cookiecutter.project_name}}/README.md | 0
.../docker/{develop => deploy}/daemon/Dockerfile | 76 +++--
.../docker/deploy/daemon/deps/start.sh | 2 +
.../docker/develop/daemon/Dockerfile | 76 +++--
.../{{cookiecutter.project_name}}/docs.yaml | 0
.../{{cookiecutter.project_name}}/engine.messages | 0
.../{{cookiecutter.project_name}}/engine.metadata | 0
.../{{cookiecutter.project_name}}/engine.params | 0
.../feedback.messages | 0
.../{{cookiecutter.project_name}}/marvin.ini | 0
.../notebooks/sample.ipynb | 0
.../{{cookiecutter.project_name}}/pytest.ini | 0
.../scripts/install_automl.sh | 0
.../{{cookiecutter.project_name}}/setup.cfg | 0
.../{{cookiecutter.project_name}}/setup.py | 0
.../tests/conftest.py | 0
.../data_handler/test_acquisitor_and_cleaner.py | 0
.../tests/data_handler/test_training_preparator.py | 0
.../tests/prediction/test_feedback.py | 0
.../tests/prediction/test_prediction_preparator.py | 0
.../tests/prediction/test_predictor.py | 0
.../tests/training/test_metrics_evaluator.py | 0
.../tests/training/test_trainer.py | 0
.../template/{{cookiecutter.project_name}}/tox.ini | 0
.../{{cookiecutter.project_package}}/VERSION | 0
.../{{cookiecutter.project_package}}/__init__.py | 0
.../_compatibility.py | 0
.../{{cookiecutter.project_package}}/_logging.py | 0
.../data_handler/__init__.py | 0
.../data_handler/acquisitor_and_cleaner.py | 0
.../data_handler/training_preparator.py | 0
.../prediction/__init__.py | 0
.../prediction/feedback.py | 0
.../prediction/prediction_preparator.py | 0
.../prediction/predictor.py | 0
.../training/__init__.py | 0
.../training/metrics_evaluator.py | 0
.../training/trainer.py | 0
marvin-cli/marvin_cli/management/test.py | 30 +-
marvin-cli/marvin_cli/utils/__init__.py | 0
marvin-cli/marvin_cli/utils/api.py | 72 ++++
marvin-cli/marvin_cli/utils/benchmark.py | 14 +-
marvin-cli/marvin_cli/utils/config.py | 0
marvin-cli/marvin_cli/utils/docker.py | 260 ++++++++++++--
marvin-cli/marvin_cli/utils/git.py | 4 +-
marvin-cli/marvin_cli/utils/log.py | 29 +-
marvin-cli/marvin_cli/utils/misc.py | 173 +++++++---
marvin-cli/pytest.ini | 0
marvin-cli/setup.py | 9 +-
.../tests/communication/test_remote_calls.py | 0
marvin-cli/tests/management/test_edit.py | 0
marvin-cli/tests/management/test_engine.py | 0
marvin-cli/tests/utils/test_benchmark.py | 0
marvin-cli/tests/utils/test_config.py | 0
marvin-cli/tests/utils/test_docker.py | 0
marvin-cli/tests/utils/test_git.py | 0
marvin-cli/tests/utils/test_misc.py | 0
marvin-cli/tox.ini | 0
91 files changed, 1161 insertions(+), 299 deletions(-)
diff --git a/marvin-cli/MANIFEST.in b/marvin-cli/MANIFEST.in
old mode 100644
new mode 100755
index 38394a0..1c905b6
--- a/marvin-cli/MANIFEST.in
+++ b/marvin-cli/MANIFEST.in
@@ -1,3 +1,5 @@
include README.md
include marvin_cli/VERSION
graft marvin_cli/management/template
+graft marvin_cli/management/airflow_template
+graft marvin_cli/management/kubernetes_template
diff --git a/marvin-cli/Makefile b/marvin-cli/Makefile
old mode 100644
new mode 100755
diff --git a/marvin-cli/README.md b/marvin-cli/README.md
old mode 100644
new mode 100755
diff --git a/marvin-cli/bin/marvin b/marvin-cli/bin/marvin
old mode 100644
new mode 100755
index 0a1b554..6bafc47
--- a/marvin-cli/bin/marvin
+++ b/marvin-cli/bin/marvin
@@ -17,8 +17,8 @@
import os
import click
-from marvin_cli.utils.misc import package_to_name
-from marvin_cli.utils.misc import kill_persisted_process
+from marvin_cli.utils.misc import package_to_name, create_tmp_marvin_folder
+from marvin_cli.utils.misc import retrieve_tmp_info
from marvin_cli.utils.config import parse_ini
from marvin_cli.utils.config import read_cli_conf, generate_default_conf
from marvin_cli.utils.docker import search_engine_container
@@ -29,18 +29,19 @@ from marvin_cli.management.engine import cli as cli_engine
from marvin_cli.management.notebook import cli as cli_notebook
from marvin_cli.management.edit import cli as cli_edit
from marvin_cli.management.test import cli as cli_test
+from marvin_cli.management.generate import cli as cli_generate
EXCLUDE_BY_TYPE = {
'toolbox': ['engine-dryrun', 'engine-grpcserver', 'engine-httpserver', 'notebook',
'lab', 'project-export', 'engine-logs', 'edit-metadata', 'test', 'test-tdd',
- 'test-tox', 'engine-bumpversion', 'benchmark', 'benchmark-plot'],
- 'engine': ['project-import']
+ 'test-tox', 'engine-bumpversion', 'benchmark', 'benchmark-plot', 'engine',
+ 'data', 'push', 'stop', 'clone', 'kube-deployment'],
+ 'engine': ['project-import', 'setup']
}
ENVIRONMENT_VARIABLES = {
'MARVIN_HOME': "{0}/marvin".format(os.environ['HOME']),
- 'MARVIN_DATA_PATH': "{0}/marvin/data".format(os.environ['HOME']),
- 'MARVIN_LOG': "{0}/marvin/data/.logs".format(os.environ['HOME'])
+ 'MARVIN_DATA_PATH': "{0}/marvin/data".format(os.environ['HOME'])
}
def _define_environ():
@@ -49,26 +50,18 @@ def _define_environ():
os.environ[var] = ENVIRONMENT_VARIABLES[var]
def _create_folders():
- _exports_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
_conf_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf')
_artifacts_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], '.artifacts')
+ _exports_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
_paths_list = [os.environ['MARVIN_HOME'], os.environ['MARVIN_DATA_PATH'],
- os.environ['MARVIN_LOG'], _exports_folder, _conf_folder, _artifacts_folder]
+ _exports_folder, _conf_folder, _artifacts_folder]
for path in _paths_list:
if not os.path.exists(path):
os.makedirs(path)
-def _initial_check_engine_mode(engine_package):
- if not search_engine_images(engine_package):
- if click.confirm('Do you want to create the engine docker image?', default=True):
- create_engine_image(engine_package)
- if not search_engine_container(engine_package):
- if click.confirm('Do you want to create the engine docker container?', default=True):
- create_daemon_container(engine_package, package_to_name(engine_package))
-
-def _read_config(engine_package):
+def _read_config():
filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
if not os.path.isfile(filepath):
conf = generate_default_conf()
@@ -83,60 +76,48 @@ sigh = """___ ___ _ _____ _ _____
| | | | (_| | | \ V /| | | | | | \__/\| |_____| |_
\_| |_/\__,_|_| \_/ |_|_| |_| \____/\_____/\___/\n\n"""
-ini_path = os.path.join(os.getcwd(), 'marvin.ini')
-config = None
-package_name = None
+engine_name = None
exclude = None
print(sigh)
_define_environ()
_create_folders()
+create_tmp_marvin_folder()
-if os.path.exists(ini_path):
- config = parse_ini(ini_path)
- package_name = config['marvin_package']
+engine = retrieve_tmp_info('engine')
+if engine is not None:
exclude = EXCLUDE_BY_TYPE['engine']
- print("[+] engine mode - {}".format(package_name))
- _initial_check_engine_mode(package_name)
+ engine_name = engine
+ print("[+] engine mode - {}".format(engine_name))
print("\n")
else:
exclude = EXCLUDE_BY_TYPE['toolbox']
print("[+] toolbox mode\n")
-config = _read_config(package_name)
+config = _read_config()
@click.group('main')
@click.pass_context
def cli(ctx):
ctx.obj = {
- 'package_name': package_name,
+ 'engine_name': engine_name,
'editor': config['editor'],
'default_host': config['default_host'],
'executor_url': config['executor_url']
}
-@cli.command("system-kill", help="Kill all marvin cli background process.")
-def kill():
- kill_persisted_process()
-
-
commands = {}
commands.update(cli_engine.commands)
commands.update(cli_notebook.commands)
commands.update(cli_edit.commands)
commands.update(cli_test.commands)
+commands.update(cli_generate.commands)
for name, command in commands.items():
if name not in exclude:
cli.add_command(command, name=name)
-cli()
-
-
-
-
-
-
+cli()
\ No newline at end of file
diff --git a/marvin-cli/bin/marvin-api b/marvin-cli/bin/marvin-api
new file mode 100644
index 0000000..2d42614
--- /dev/null
+++ b/marvin-cli/bin/marvin-api
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import marvin_cli.utils.api as api
+
+@click.group('main')
+def cli():
+ pass
+
+@cli.command("acquisitor")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def acquisitor(host, wait):
+ print("{0}/{1}".format(host, 'aq'))
+ api.run_acquisitor(host, wait)
+
+@cli.command("tpreparator")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def tpreparator(host, wait):
+ api.run_tpreparator(host, wait)
+
+@cli.command("trainer")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def trainer(host, wait):
+ api.run_trainer(host, wait)
+
+@cli.command("evaluator")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def evaluator(host, wait):
+ api.run_evaluator(host, wait)
+
+cli()
\ No newline at end of file
diff --git a/marvin-cli/bin/marvin_complete b/marvin-cli/bin/marvin_complete
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/VERSION b/marvin-cli/marvin_cli/VERSION
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/__init__.py b/marvin-cli/marvin_cli/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/__init__.py b/marvin-cli/marvin_cli/communication/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/remote_calls.py b/marvin-cli/marvin_cli/communication/remote_calls.py
old mode 100644
new mode 100755
index 9a11969..a9d7836
--- a/marvin-cli/marvin_cli/communication/remote_calls.py
+++ b/marvin-cli/marvin_cli/communication/remote_calls.py
@@ -27,13 +27,13 @@ class RemoteError(Exception):
pass
COMMANDS = {
- 'DRYRUN': daemon_pb2.Command.CommandType.DRYRUN,
- 'TEST': daemon_pb2.Command.CommandType.TEST,
- 'GRPC': daemon_pb2.Command.CommandType.GRPC,
- 'TDD': daemon_pb2.Command.CommandType.TDD,
- 'TOX': daemon_pb2.Command.CommandType.TOX,
- 'NOTEBOOK': daemon_pb2.Command.CommandType.NOTEBOOK,
- 'LAB': daemon_pb2.Command.CommandType.LAB
+ 'DRYRUN': daemon_pb2.Command.CommandType.Value('DRYRUN'),
+ 'TEST': daemon_pb2.Command.CommandType.Value('TEST'),
+ 'GRPC': daemon_pb2.Command.CommandType.Value('GRPC'),
+ 'TDD': daemon_pb2.Command.CommandType.Value('TDD'),
+ 'TOX': daemon_pb2.Command.CommandType.Value('TOX'),
+ 'NOTEBOOK': daemon_pb2.Command.CommandType.Value('NOTEBOOK'),
+ 'LAB': daemon_pb2.Command.CommandType.Value('LAB')
}
class RemoteCalls:
@@ -47,7 +47,7 @@ class RemoteCalls:
def call_command(self, name, parameters):
call = daemon_pb2.Command(command=COMMANDS[name], parameters=parameters)
response = self.stub.callCommand(call)
- if response.status == daemon_pb2.Status.NOK:
+ if response.status == daemon_pb2.Status.StatusType.Value('NOK'):
raise RemoteError("Error during {}.".format(name))
else:
logger.info("{} triggered!".format(name))
@@ -55,7 +55,7 @@ class RemoteCalls:
def stop_command(self, name):
call = daemon_pb2.Interruption()
response = self.stub.stopCommand(call)
- if response.status == daemon_pb2.Status.NOK:
+ if response.status == daemon_pb2.Status.StatusType.Value('NOK'):
raise RemoteError("Error during stop {}.".format(name))
else:
logger.info("{} stopped!".format(name))
diff --git a/marvin-cli/marvin_cli/communication/stubs/__init__.py b/marvin-cli/marvin_cli/communication/stubs/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/__init__.py b/marvin-cli/marvin_cli/management/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json b/marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json
new file mode 100644
index 0000000..d8c194e
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json
@@ -0,0 +1,8 @@
+{
+ "dag_name": "marvin-dag",
+ "host": "http://localhost:8000",
+ "owner": "marvin",
+ "interval_days": "1",
+ "timeout": "60",
+ "retries": "3"
+}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py b/marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py
new file mode 100644
index 0000000..1ada7fe
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py
@@ -0,0 +1,53 @@
+import airflow
+from airflow import DAG
+from airflow.operators.bash_operator import BashOperator
+from datetime import timedelta
+
+HOST = "{{cookiecutter.host}}"
+
+args = {
+ 'owner': '{{cookiecutter.owner}}',
+}
+
+dag = DAG(
+ dag_id='{{cookiecutter.interval_days}}',
+ default_args=args,
+ schedule_interval=timedelta(days={{cookiecutter.interval_days}}),
+ dagrun_timeout=timedelta(minutes={{cookiecutter.timeout}})
+)
+
+acquisitor = BashOperator(
+ task_id='acquisitor',
+ bash_command='marvin-api acquisitor --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
+
+tpreparator = BashOperator(
+ task_id='tpreparator',
+ bash_command='marvin-api tpreparator --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
+
+trainer = BashOperator(
+ task_id='trainer',
+ bash_command='marvin-api trainer --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
+
+evaluator = BashOperator(
+ task_id='evaluator',
+ bash_command='marvin-api evaluator --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/edit.py b/marvin-cli/marvin_cli/management/edit.py
old mode 100644
new mode 100755
index 37b24b6..156c748
--- a/marvin-cli/marvin_cli/management/edit.py
+++ b/marvin-cli/marvin_cli/management/edit.py
@@ -27,9 +27,3 @@ def cli():
def config(ctx):
filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
os.system(ctx.obj['editor'] + ' ' + filepath)
-
-@cli.command("edit-metadata", help="Edit engine.metadata.")
-@click.pass_context
-def metadata(ctx):
- filepath = "engine.metadata"
- os.system(ctx.obj['editor'] + ' ' + filepath)
diff --git a/marvin-cli/marvin_cli/management/engine.py b/marvin-cli/marvin_cli/management/engine.py
old mode 100644
new mode 100755
index 00ab26f..14b1925
--- a/marvin-cli/marvin_cli/management/engine.py
+++ b/marvin-cli/marvin_cli/management/engine.py
@@ -23,17 +23,26 @@ import wget
import click
import pickle
import pathlib
+import shutil
import subprocess
+import getpass
from cookiecutter.main import cookiecutter
from shutil import which
+from ..utils.docker import DaemonManagement
+from ..utils.docker import search_engine_container, search_docker_volume, shutdown_and_delete_container
+from ..utils.docker import create_engine_image, create_deploy_image_and_push, create_docker_volume, create_daemon_container
+from ..utils.docker import create_executor_container, delete_image_and_volume
+from ..utils.docker import rename_image, create_tfserving_container
from ..communication.remote_calls import RemoteCalls
from ..utils.misc import package_folder, extract_folder, get_version
from ..utils.misc import call_logs, package_to_name, get_executor_path_or_download
-from ..utils.misc import generate_timestamp, persist_process
+from ..utils.misc import generate_timestamp, write_tmp_info, generate_keys
+from ..utils.misc import init_port_forwarding
from ..utils.git import git_init, bump_version
from ..utils.log import get_logger
from ..utils.benchmark import benchmark_thread, create_poi, make_graph
+
logger = get_logger('engine')
@click.group("engine")
@@ -45,42 +54,155 @@ def _validate_project_name(name):
TEMPLATE_FOLDER = os.path.join(pathlib.Path(__file__).parent.absolute(), 'template')
+def check_engine(engine):
+ _engine_path = os.path.join(os.environ['MARVIN_HOME'], engine)
+ daemon = DaemonManagement(engine)
+
+ if os.path.exists(_engine_path):
+ if click.confirm('Do you want to update the cached engine?', default=True):
+ shutil.rmtree(_engine_path)
+ daemon.clone_engine()
+ else:
+ logger.info("Caching engine...")
+ daemon.clone_engine()
+ logger.info("Caching engine... Done!")
+
@cli.command("project-generate", help="Generate engine project")
@click.option('--name', '-n', prompt='Project name', help='Engine name')
-@click.option('--description', '-d', prompt='Project description', help='Engine description')
+@click.option('--description', '-d', prompt='Project description', help='Engine description', default='Marvin project')
@click.option('--url', '-u', prompt='Project URL', help='Engine URL Address', default='marvin.apache.org')
-@click.option('--maintainer', '-m', prompt='Maintainer', help='Engine Maintainer')
-@click.option('--email', '-e', prompt='Maintainer E-mail', help='Engine maintainer e-mail')
-@click.option('--dest', '-f', envvar='MARVIN_HOME', type=click.Path(exists=True), help='Root folder path for the creation')
+@click.option('--maintainer', '-m', prompt='Maintainer', help='Engine Maintainer', default='Marvin-AI')
+@click.option('--email', '-e', prompt='Maintainer E-mail', help='Engine maintainer e-mail', default='dev@marvin.apache.org')
+@click.option(
+ '--ptype',
+ '-pt',
+ default='python',
+ type=click.Choice(['python', 'tfx']),
+ help='Project type: Regular python project or TFX.')
@click.option('--template', '-t', help='Base template for engine.', default=TEMPLATE_FOLDER, type=click.Path(exists=True))
-def generate(name, description, url, maintainer, email, dest, template):
+def generate(name, description, url, maintainer, email, ptype, template):
+ #create engine files in /tmp/marvin
+ _dest = '/tmp/marvin'
_processed_name = _validate_project_name(name)
-
_extras_dir = {
'project_name': _processed_name,
'project_package': 'marvin_' + _processed_name.lower(),
'project_url': url,
'project_description': description,
'maintainer_name': maintainer,
- 'maintainer_email': email
+ 'maintainer_email': email,
+ 'project_type': ptype
}
- print(TEMPLATE_FOLDER)
- _init_dir = os.path.join(dest, _processed_name)
- cookiecutter(template, output_dir=dest, extra_context=_extras_dir, no_input=True)
+ _init_dir = os.path.join(_dest, _processed_name)
+ cookiecutter(template, output_dir=_dest, extra_context=_extras_dir, no_input=True)
git_init(_init_dir)
- logger.info("Engine {0} created".format(_processed_name))
+ #generate and put key in docker context
+ _pubkey_path = generate_keys(_processed_name)
+ _new_pubkey_path = os.path.join(_init_dir, 'docker', 'develop', 'daemon', 'id_rsa.pub')
+ shutil.move(_pubkey_path, _new_pubkey_path)
+
+ logger.info("Engine {0} created in /tmp/marvin.".format(_processed_name))
+ create_engine_image(name, _init_dir)
+ logger.info("Removing temporary files.")
+ shutil.rmtree(_init_dir, ignore_errors=True)
+ logger.info("Engine creation done!")
EXPORT_PATH = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
+@cli.command("clone", help="Clone files from daemon container.")
+@click.pass_context
+def engine_clone(ctx):
+ daemon = DaemonManagement(ctx.obj['engine_name'])
+ daemon.clone_engine()
+ daemon.clone_artifacts()
+ daemon.clone_logs()
+
+@cli.command("push", help="Rewrite engine files of daemon container.")
+@click.option('--compress/--not-compress', '-c/-nc', default=True, is_flag=True, help='Compress the stream.')
+@click.pass_context
+def engine_push(ctx, compress):
+ daemon = DaemonManagement(ctx.obj['engine_name'])
+ daemon.push_engine(compress)
+
+@cli.command("data", help="Actions related to data folder of daemon container.")
+@click.option(
+ '--action',
+ '-a',
+ default='list',
+ type=click.Choice(['push', 'delete', 'list']),
+ help='Data folder action type')
+@click.option('--compress/--not-compress', '-c/-nc', default=True, is_flag=True, help='Compress the stream.')
+@click.pass_context
+def data_push(ctx, action, compress):
+ daemon = DaemonManagement(ctx.obj['engine_name'])
+ if action == 'list':
+ daemon.list_data_files()
+ elif action == 'push':
+ _path = click.prompt('File to push', type=click.Path(exists=True))
+ daemon.push_data(_path, compress)
+ else:
+ _path = click.prompt('File to delete')
+ daemon.delete_data(_path)
+
+@cli.command("setup", help="Setup docker container and volumes to development.")
+@click.argument('engine', nargs=1)
+@click.pass_context
+def setup(ctx, engine):
+ _engine_volume = "marvin-{}-vol".format(engine)
+
+ logger.info("Setting up engine components...")
+ if not search_docker_volume("marvin-log"):
+ create_docker_volume("marvin-log")
+ if not search_docker_volume("marvin-data"):
+ create_docker_volume("marvin-data")
+ if not search_docker_volume(_engine_volume):
+ create_docker_volume(_engine_volume)
+ if not search_engine_container(engine):
+ create_daemon_container(engine)
+ write_tmp_info('engine', engine)
+ logger.info("Setting up engine components... Done!")
+ logger.info("Attaching logs...")
+ call_logs(engine)
+ logger.info("Attaching logs... Done!")
+ logger.info("Enabling port forwarding...")
+ init_port_forwarding(engine, ctx.obj['default_host'], [50057], background=True)
+ logger.info("Enabling port forwarding... Done!")
+
+@cli.command("stop", help="Stop docker container and workon.")
+@click.option('--delete', '-d', default=False, is_flag=True, help='Delete engine files permanently.')
+@click.pass_context
+def stop(ctx, delete):
+ _container_name = "marvin-cont-{}".format(ctx.obj['engine_name'])
+ _lock_path = '/tmp/marvin/engine'
+
+ logger.info("Stopping and deleting engine container...")
+ shutdown_and_delete_container(_container_name)
+ os.remove(_lock_path)
+ logger.info("Stopping and deleting engine container... Done!")
+
+ if delete:
+ check = click.prompt('If you are sure, type the engine name: ')
+ if check == ctx.obj['engine_name']:
+ _image_name = "marvin-{}".format(ctx.obj['engine_name'])
+ _volume_name = "marvin-{}-vol".format(ctx.obj['engine_name'])
+ delete_image_and_volume(_image_name, _volume_name)
+ logger.warning("Deleting private key...")
+ _key_path = os.path.join(os.environ['MARVIN_DATA_PATH'], '.keys',
+ ctx.obj['engine_name'])
+ shutil.rmtree(_key_path)
+ logger.warning("Deleting private key... Done!")
+
@cli.command("project-export", help="Export engine project to a archive file.")
@click.option('--dest', '-d', default=EXPORT_PATH, type=click.Path(exists=True), help='Output folder.')
@click.pass_context
def export(ctx, dest):
- filename = os.path.join(dest, ctx.obj['package_name']
- + '-' + get_version(ctx.obj['package_name']) + ".tar.gz")
+ check_engine(ctx.obj['engine_name'])
+ path = os.path.join(os.environ['MARVIN_HOME'], ctx.obj['engine_name'])
+ filename = os.path.join(dest, ctx.obj['engine_name']
+ + '-' + get_version(path, ctx.obj['engine_name']) + ".tar.gz")
- package_folder(os.getcwd(), filename)
+ package_folder(path, filename)
@cli.command("project-import", help="Import engine project from archive file.")
@click.option('--file', '-f', type=click.Path(exists=True), help='Compressed Engine file.')
@@ -89,7 +211,7 @@ def import_project(file, dest):
extract_folder(file, dest)
@cli.command("engine-dryrun", help="Run engines in a standalone way.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option(
'--action',
@@ -98,12 +220,28 @@ def import_project(file, dest):
type=click.Choice(['all', 'acquisitor', 'tpreparator', 'trainer', 'evaluator', 'ppreparator', 'predictor', 'feedback']),
help='Marvin engine action name')
@click.option('--profiling', '-p', default=False, is_flag=True, help='Deterministic profiling of user code.')
-def dryrun(grpchost, grpcport, action, profiling):
+@click.pass_context
+def dryrun(ctx, grpchost, grpcport, action, profiling):
+ if not grpchost:
+ grpchost = 'localhost'
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_dryrun(action, profiling)
+def grpc_port_forwarding(engine_name, grpchost):
+ ports = [
+ 50051,
+ 50052,
+ 50053,
+ 50054,
+ 50055,
+ 50056,
+ ]
+ init_port_forwarding(engine_name, grpchost,
+ ports_list=ports)
+
@cli.command("engine-grpcserver", help="Run gRPC of given actions.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option(
'--action',
@@ -113,31 +251,22 @@ def dryrun(grpchost, grpcport, action, profiling):
help='Marvin engine action name')
@click.option('--max-workers', '-w', help='Max Workers', default=None)
@click.option('--max-rpc-workers', '-rw', help='Max gRPC Workers', default=None)
-def grpc(grpchost, grpcport, action, max_workers, max_rpc_workers):
+@click.pass_context
+def grpc(ctx, grpchost, grpcport, action, max_workers, max_rpc_workers):
+ if not grpchost:
+ grpchost = 'localhost'
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_grpc(action, max_workers, max_rpc_workers)
- try:
- while(True):
- time.sleep(100)
- except KeyboardInterrupt:
- rc.stop_grpc()
- logger.info("gRPC server terminated!")
-
-@cli.command("engine-logs", help="Show daemon execution.")
-@click.option('--follow', '-f', is_flag=True)
-@click.option('--tail', '-t', default=True, is_flag=True)
-@click.option('--buffer', '-b', default=20)
-@click.pass_context
-def docker_logs(ctx, follow, tail, buffer):
- p_logs = call_logs(ctx.obj['package_name'], follow, buffer)
- if follow:
- persist_process(p_logs)
+ grpc_port_forwarding(ctx.obj['engine_name'], ctx.obj['default_host'])
+ rc.stop_grpc()
+ logger.info("gRPC server terminated!")
@cli.command("engine-httpserver", help="Run executor HTTP server.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
-@click.option('--host', '-h', prompt='API host', help='REST API Host', default='localhost')
-@click.option('--port', '-p', prompt='API port', help='REST API Port', default='8000')
+@click.option('--host', '-h', help='REST API Host', default='localhost')
+@click.option('--port', '-p', help='REST API Port', default='8000')
@click.option('--protocol', '-pr', help='Marvin protocol to be loaded during initialization.', default='')
@click.option(
'--action',
@@ -150,9 +279,16 @@ def docker_logs(ctx, follow, tail, buffer):
@click.option('--executor-path', '-e', help='Marvin engine executor jar path', type=click.Path(exists=True))
@click.option('--extra-executor-parameters', '-jvm', help='Use to send extra JVM parameters to engine executor process')
@click.option('--benchmark', '-b', default=False, is_flag=True, help='Run benchmark.')
+@click.option('--no-docker', '-nd', default=False, is_flag=True, help='Don\'t run the engine-executor on a Docker container.')
@click.pass_context
def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
- max_rpc_workers, executor_path, extra_executor_parameters, benchmark):
+ max_rpc_workers, executor_path, extra_executor_parameters, benchmark, no_docker):
+
+ if not grpchost:
+ grpchost = 'localhost'
+
+ if not host:
+ host = ctx.obj['default_host']
rc = RemoteCalls(grpchost, grpcport)
@@ -164,51 +300,77 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
sys.exit(1)
bench_thread = None
+ httpserver = None
try:
- if not executor_path:
- executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
-
- command_list = ['java']
- command_list.append('-DmarvinConfig.engineHome={}'.format(os.getcwd()))
- command_list.append('-DmarvinConfig.ipAddress={}'.format(host))
- command_list.append('-DmarvinConfig.port={}'.format(port))
- command_list.append('-DmarvinConfig.protocol={}'.format(protocol))
-
- if extra_executor_parameters:
- command_list.append(extra_executor_parameters)
-
- command_list.append('-jar')
- command_list.append(executor_path)
-
- if benchmark:
- logger.info("Init benchmark...")
- timestamp = generate_timestamp()
- bench_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
- bench_thread.start()
-
- httpserver = subprocess.Popen(command_list)
-
- except:
+ if not no_docker:
+ create_executor_container(ctx.obj['engine_name'])
+ else:
+ if not executor_path:
+ executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
+
+ check_engine(ctx.obj['engine_name'])
+ engine_path = os.path.join(os.environ['MARVIN_HOME'], ctx.obj['engine_name'])
+
+ command_list = ['java']
+ command_list.append('-DmarvinConfig.engineHome={}'.format(engine_path))
+ command_list.append('-DmarvinConfig.ipAddress={}'.format(host))
+ command_list.append('-DmarvinConfig.port={}'.format(port))
+ command_list.append('-DmarvinConfig.protocol={}'.format(protocol))
+
+ if extra_executor_parameters:
+ command_list.append(extra_executor_parameters)
+
+ command_list.append('-jar')
+ command_list.append(executor_path)
+
+ if benchmark:
+ logger.info("Init benchmark...")
+ timestamp = generate_timestamp()
+ bench_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
+ bench_thread.start()
+
+ httpserver = subprocess.Popen(command_list)
+
+ while True:
+ try:
+ time.sleep(100)
+ except KeyboardInterrupt:
+ break
+
+ except Exception as e:
logger.error("Could not start http server!")
+ if not no_docker:
+ shutdown_and_delete_container("marvin-executor-" +
+ ctx.obj['engine_name'])
+ print(e)
rc.stop_grpc()
if benchmark:
bench_thread.terminate()
sys.exit(1)
-
- try:
- while True:
- time.sleep(100)
-
- except KeyboardInterrupt:
+
+ if no_docker:
logger.info("Terminating http and grpc servers...")
rc.stop_grpc()
httpserver.terminate() if httpserver else None
logger.info("Http and grpc servers terminated!")
- if benchmark:
- bench_thread.terminate()
+ else:
+ try:
+ while True:
+ time.sleep(100)
+
+ except KeyboardInterrupt:
+ logger.info("Terminating http and grpc servers...")
+ rc.stop_grpc()
+ shutdown_and_delete_container("marvin-executor-" +
+ ctx.obj['engine_name'])
+ logger.info("Http and grpc servers terminated!")
+
+ if benchmark:
+ bench_thread.terminate()
logger.info("Benchmark terminated!")
- sys.exit(0)
+
+ sys.exit(0)
@cli.command('engine-bumpversion', help='Bump, commit and tag engine version.')
@@ -220,8 +382,11 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
default='patch',
type=click.Choice(['major', 'minor', 'patch']),
help='The part of the version to increase.')
-def bumpversion(verbose, dry_run, part):
- bump_version(part, verbose, dry_run)
+@click.pass_context
+def bumpversion(ctx, verbose, dry_run, part):
+ check_engine(ctx.obj['engine_name'])
+ path = os.path.join(os.environ['MARVIN_HOME'], ctx.obj['engine_name'])
+ bump_version(path, part, verbose, dry_run)
POI_LABELS = {
'acquisitor': 'ac',
@@ -235,16 +400,20 @@ def _sleep(sec):
time.sleep(5)
@cli.command("benchmark", help="Collect engine benchmark stats.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--profiling', '-p', default=False, is_flag=True, help='Deterministic profiling of user code.')
@click.option('--delay', '-d', default=False, is_flag=True, help='Delay the benchmark for 5 seconds.')
@click.pass_context
def btest(ctx, grpchost, grpcport, profiling, delay):
timestamp = generate_timestamp()
- b_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
+ b_thread = benchmark_thread(ctx.obj['engine_name'], timestamp)
actions = ['acquisitor', 'tpreparator', 'trainer',
'evaluator']
+
+ if not grpchost:
+ grpchost = 'localhost'
+
rc = RemoteCalls(grpchost, grpcport)
initial_time = time.time()
b_thread.start()
@@ -303,3 +472,61 @@ def plot(protocol):
make_graph(op_name, label, protocol)
except KeyboardInterrupt:
sys.exit(0)
+
+@cli.command("kube-deployment", help="Deploy Kubernetes production pod.")
+@click.option('--namespace', '-p', default='default', help='Define Kubernetes namespace.')
+@click.option('--nreplicas', '-nr', default=1, help='Define number of pod replicas.')
+@click.option('--name-in-registry', '-nr', prompt='Image name in registry', help='Docker registry repository to pull the deploy image.', default='')
+@click.option('--target-port', '-tp', help='Target port on cluster.', default="9736")
+@click.pass_context
+def kube_deploy(ctx, namespace, nreplicas, name_in_registry, target_port):
+ engine_name = ctx.obj['engine_name']
+ port = None
+ if not click.confirm('Do you want to use the TFX serving method?', default=False):
+ artifact_path = os.path.join(os.environ['MARVIN_DATA_PATH'],
+ '.artifacts', engine_name)
+ engine_path = os.path.join(os.environ['MARVIN_HOME'], engine_name)
+ deps_path = os.path.join(engine_path, 'docker', 'deploy', 'daemon', 'deps')
+ executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
+
+ daemon = DaemonManagement(engine_name)
+ check_engine(engine_name)
+ daemon.clone_artifacts()
+
+ shutil.move(artifact_path, os.path.join(deps_path, 'artifacts'))
+ shutil.copy(executor_path, deps_path)
+ create_deploy_image_and_push(engine_name, engine_path, name_in_registry)
+ port = "8000"
+ else:
+ model_path = click.prompt('Model path', type=str)
+ old_image_name = "tfserving:{}".format(engine_name)
+ create_tfserving_container(engine_name, model_path)
+ rename_image(old_image_name, name_in_registry)
+ logger.info("Pushing deploy image to registry...")
+ os.system("docker push {}".format(name_in_registry))
+ logger.info("Pushing deploy image to registry... Done!")
+ port = "8500"
+
+ kubernetes_template = os.path.join(pathlib.Path(__file__).parent.absolute(),
+ 'kubernetes_template')
+
+ _extras_dir = {
+ 'engine_name': engine_name,
+ 'n_reps': nreplicas,
+ 'image_name': name_in_registry,
+ 'service_name': "{}-service".format(engine_name),
+ 'deployment_name': "{}-deployment".format(engine_name),
+ 'target_port': target_port,
+ 'container_port': port
+ }
+
+ cookiecutter(kubernetes_template, output_dir=os.getcwd(),
+ extra_context=_extras_dir, no_input=True)
+
+
+
+
+
+
+
+
diff --git a/marvin-cli/marvin_cli/management/generate.py b/marvin-cli/marvin_cli/management/generate.py
new file mode 100644
index 0000000..0d70764
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/generate.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import click
+import pathlib
+from cookiecutter.main import cookiecutter
+from ..utils.log import get_logger
+
+logger = get_logger('management.generate')
+
+@click.group("generate")
+def cli():
+ pass
+
+AIRFLOW_TEMPLATE = os.path.join(pathlib.Path(__file__).parent.absolute(), 'airflow_template')
+
+@cli.command("generate-dag", help="Generate Airflow DAG.")
+@click.option('--name', '-n', prompt='DAG name', default='marvin-dag', help='DAG name')
+@click.option('--host', '-h', prompt='Host', default='http://localhost:8000',
+ help='Hostname with port.')
+@click.option('--owner', '-o', prompt='Owner', default='Marvin',
+ help='Owner of the DAG.')
+@click.option('--interval-days', '-i', prompt='Interval days', default='1',
+ help='DAG executiom interval in days.')
+@click.option('--timeout', '-t', prompt='Timeout', default='60',
+ help='Timeout in minutes.')
+@click.option('--retries', '-r', prompt='Retries', default='3',
+ help='# of task each retries.')
+@click.option('--path', '-p', help='Output path to file.',
+ default=os.getcwd(), type=click.Path(exists=True))
+def generate_dag(name, host, owner, interval_days, timeout, retries, path):
+ _dest = path
+ _extras_dir = {
+ 'dag_name': name,
+ 'host': host,
+ 'owner': owner,
+ 'interval_days': interval_days,
+ 'timeout': timeout,
+ 'retries': retries
+ }
+ cookiecutter(AIRFLOW_TEMPLATE, output_dir=_dest, extra_context=_extras_dir, no_input=True)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json b/marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json
new file mode 100755
index 0000000..b6bb8f7
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json
@@ -0,0 +1,9 @@
+{
+ "engine_name": "hello",
+ "n_reps": "1",
+ "image_name": "marvin_deployment",
+ "service_name": "marvin_service",
+ "deployment_name": "marvin_deployment",
+ "target_port": "9736",
+ "container_port": "8000"
+}
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh
new file mode 100644
index 0000000..4f23d69
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+kubectl apply -f $SCRIPTPATH/{{cookiecutter.engine_name}}_deployment.yaml
+kubectl apply -f $SCRIPTPATH/{{cookiecutter.engine_name}}_service.yaml
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml
new file mode 100644
index 0000000..47e91fb
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{cookiecutter.deployment_name}}
+ labels:
+ app: {{cookiecutter.deployment_name}}
+spec:
+ replicas: {{cookiecutter.n_reps}}
+ selector:
+ matchLabels:
+ app: {{cookiecutter.deployment_name}}
+ template:
+ metadata:
+ labels:
+ app: {{cookiecutter.deployment_name}}
+ spec:
+ containers:
+ - name: {{cookiecutter.deployment_name}}
+ image: {{cookiecutter.image_name}}
+ ports:
+ - containerPort: {{cookiecutter.container_port}}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml
new file mode 100644
index 0000000..9f049bf
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{cookiecutter.service_name}}
+spec:
+ selector:
+ app: {{cookiecutter.deployment_name}}
+ ports:
+ - protocol: TCP
+ port: {{cookiecutter.container_port}}
+ targetPort: {{cookiecutter.target_port}}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/notebook.py b/marvin-cli/marvin_cli/management/notebook.py
old mode 100644
new mode 100755
index 01a8488..820743d
--- a/marvin-cli/marvin_cli/management/notebook.py
+++ b/marvin-cli/marvin_cli/management/notebook.py
@@ -15,25 +15,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import sys
import click
+import time
+from ..utils.log import get_logger
from ..communication.remote_calls import RemoteCalls
+from ..utils.misc import init_port_forwarding
+
+logger = get_logger('management.notebook')
@click.group("notebook")
def cli():
pass
@cli.command("notebook", help="Run custom engine Jupyter Notebook.")
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
-@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
-def notebook(grpchost, grpcport, notebook_port):
- rc = RemoteCalls(grpchost, grpcport)
- rc.run_notebook(notebook_port)
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
+@click.option('--notebook-port', '-np', help='Notebook port', default='8888')
+@click.option('--no-port-forwarding', '-npf', is_flag=True, default=False,
+ help='Connect ports between this system and remote host with SSH tunnel')
+@click.pass_context
+def notebook(ctx, grpchost, grpcport, notebook_port, no_port_forwarding):
+ if not grpchost:
+ grpchost = 'localhost'
-@cli.command("lab", help="Run custom engine Jupyter Lab.")
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
-@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
-def lab(grpchost, grpcport, notebook_port):
rc = RemoteCalls(grpchost, grpcport)
- rc.run_lab(notebook_port)
+ rc.run_notebook(notebook_port)
+ if not no_port_forwarding:
+ time.sleep(5)
+ logger.info("Enabling port forwarding {0}:{0}...".format(notebook_port))
+ init_port_forwarding(ctx.obj['engine_name'], ctx.obj['default_host'],
+ ports_list=[int(notebook_port)])
+ sys.exit(0)
diff --git a/marvin-cli/marvin_cli/management/template/cookiecutter.json b/marvin-cli/marvin_cli/management/template/cookiecutter.json
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
old mode 100644
new mode 100755
index c6c58b1..a747779
--- a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
@@ -6,4 +6,5 @@ include README.md
include {{cookiecutter.project_package}}/VERSION
recursive-include notebooks *
prune notebooks/build
-recursive-include tests *
\ No newline at end of file
+recursive-include tests *
+include docker/deploy/daemon/deps/start.sh
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/Dockerfile
similarity index 54%
copy from marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
copy to marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/Dockerfile
index 8418ff2..6e5deb3 100644
--- a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/Dockerfile
@@ -4,32 +4,44 @@
FROM marvin-daemon:python
-MAINTAINER {{cookiecutter.maintainer_name}}
-
ENV SLEEP_MILLIS 0
+#############################################################
+# CREATE USER #
+#############################################################
+
+RUN useradd -m marvin
+
##############################################################
# Define all environment variables to be used
##############################################################
-ENV ENGINE={{cookiecutter.project_name}}
-ENV MARVIN_HOME=/opt/marvin
-ENV MARVIN_LOG=$MARVIN_HOME/log
-ENV MARVIN_DATA_PATH=/opt/marvin/data
+ENV MARVIN_HOME=/home/marvin/
+ENV MARVIN_LOG=/home/marvin/log
+ENV MARVIN_DATA_PATH=/home/marvin/data
ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
ENV SPARK_HOME=/opt/spark
+ENV MARVIN_ENGINE_NAME={{cookiecutter.project_name}}
ENV SPARK_CONF_DIR=$SPARK_HOME/conf
ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
ENV YARN_CONF_DIR=$SPARK_CONF_DIR
##############################################################
+# Put here the artifact protocol #
+ENV ARTIFACT=
+##############################################################
+
+
+ENV PYTHONUNBUFFERED=0
+
+##############################################################
# Create all folders needed
##############################################################
RUN mkdir -p $MARVIN_HOME && \
mkdir -p $MARVIN_DATA_PATH && \
- mkdir -p $MARVIN_ENGINE_HOME && \
- mkdir -p $MARVIN_LOG
+ mkdir -p $MARVIN_LOG && \
+ mkdir -p $MARVIN_DATA_PATH/.artifacts/$MARVIN_ENGINE_NAME
##############################################################
# Install Apache Spark
@@ -47,24 +59,34 @@ RUN mkdir -p $MARVIN_HOME && \
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
-# INSTALL ENGINE FROM SOURCE #
-##############################################################
-
-COPY {{cookiecutter.project_package}}-0.0.1.tar.gz .
-RUN pip install {{cookiecutter.project_package}}-0.0.1.tar.gz
-
-##############################################################
-# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+# INSTALL ENGINE FROM SOURCE #
##############################################################
-RUN pip install --no-cache pandas \
-&& pip install --no-cache matplotlib \
-&& pip install --no-cache cython \
-&& pip install --no-cache numpy \
-&& pip install --no-cache scikit-learn \
-&& pip install --no-cache pytest \
-&& pip install --no-cache pytest-cov \
-&& pip install --no-cache pytest-watch
+ADD {{cookiecutter.project_package}}-0.0.1.tar.gz $MARVIN_HOME
+ADD deps/artifacts/$MARVIN_ENGINE_NAME/* $MARVIN_DATA_PATH/.artifacts/$MARVIN_ENGINE_NAME/
+ADD deps/marvin-engine-executor-assembly-*.jar $MARVIN_DATA_PATH/executor.jar
+ADD deps/start.sh $MARVIN_DATA_PATH/start.sh
+
+RUN cd $MARVIN_HOME && \
+mv {{cookiecutter.project_name}} engine && \
+cd $MARVIN_ENGINE_HOME && \
+pip3 install -e . --no-compile && \
+apt update && \
+apt install -y openjdk-8-jre && \
+update-alternatives --config java
+
+###############################################################
+# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+###############################################################
+
+RUN pip3 install --no-cache pandas \
+&& pip3 install --no-cache matplotlib \
+&& pip3 install --no-cache cython \
+&& pip3 install --no-cache numpy \
+&& pip3 install --no-cache scikit-learn \
+&& pip3 install --no-cache pytest \
+&& pip3 install --no-cache pytest-cov \
+&& pip3 install --no-cache pytest-watch
##############################################################
# Uninstalling unnecessary software and cleaning cache
@@ -76,7 +98,7 @@ RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
# Starts the daemon
##############################################################
-EXPOSE 50051-50057
-EXPOSE 8888
+EXPOSE 50051-50056
+EXPOSE 8000
-CMD /bin/bash -c "cd $MARVIN_ENGINE_HOME && marvin-daemon"
\ No newline at end of file
+ENTRYPOINT ["/bin/bash", "-c", "cd $MARVIN_DATA_PATH && bash start.sh"]
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh
new file mode 100644
index 0000000..bfa415e
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh
@@ -0,0 +1,2 @@
+cd $MARVIN_ENGINE_HOME && nohup marvin-daemon &
+java -DmarvinConfig.engineHome=/home/marvin/engine -DmarvinConfig.ipAddress=0.0.0.0 -DmarvinConfig.port=8000 -DmarvinConfig.protocol=$PROTOCOL -jar $MARVIN_DATA_PATH/executor.jar
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
old mode 100644
new mode 100755
index 8418ff2..da3aa07
--- a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
@@ -4,23 +4,30 @@
FROM marvin-daemon:python
-MAINTAINER {{cookiecutter.maintainer_name}}
+LABEL MAINTAINER {{cookiecutter.maintainer_name}}
ENV SLEEP_MILLIS 0
+#############################################################
+# CREATE USER #
+#############################################################
+
+RUN useradd -m marvin
+
##############################################################
# Define all environment variables to be used
##############################################################
-ENV ENGINE={{cookiecutter.project_name}}
-ENV MARVIN_HOME=/opt/marvin
-ENV MARVIN_LOG=$MARVIN_HOME/log
-ENV MARVIN_DATA_PATH=/opt/marvin/data
+ENV MARVIN_HOME=/home/marvin/
+ENV MARVIN_LOG=/home/marvin/log
+ENV MARVIN_DATA_PATH=/home/marvin/data
ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
ENV SPARK_HOME=/opt/spark
+ENV MARVIN_ENGINE_NAME={{cookiecutter.project_name}}
ENV SPARK_CONF_DIR=$SPARK_HOME/conf
ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+ENV PYTHONUNBUFFERED=0
##############################################################
# Create all folders needed
@@ -28,8 +35,8 @@ ENV YARN_CONF_DIR=$SPARK_CONF_DIR
RUN mkdir -p $MARVIN_HOME && \
mkdir -p $MARVIN_DATA_PATH && \
- mkdir -p $MARVIN_ENGINE_HOME && \
- mkdir -p $MARVIN_LOG
+ mkdir -p $MARVIN_LOG && \
+ mkdir -p $MARVIN_HOME/.ssh
##############################################################
# Install Apache Spark
@@ -46,25 +53,49 @@ RUN mkdir -p $MARVIN_HOME && \
RUN mkdir -p $SPARK_CONF_DIR
+###############################################################
+# CONFIGURE SSH SERVER #
+###############################################################
+
+ADD id_rsa.pub $MARVIN_HOME/.ssh/
+RUN apt update && \
+apt install -y net-tools openssh-server sudo && \
+sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/g' /etc/ssh/sshd_config && \
+sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+sed -i 's/PrintModt yes/PrintModt no/' /etc/ssh/sshd_config && \
+cat $MARVIN_HOME/.ssh/id_rsa.pub >> $MARVIN_HOME/.ssh/authorized_keys && \
+service ssh start && \
+chown -R marvin:marvin /home/marvin && \
+chmod -R 700 $MARVIN_HOME/.ssh && \
+chmod -R 600 $MARVIN_HOME/.ssh/authorized_keys && \
+echo 'marvin:Marvin' | chpasswd && \
+addgroup marvin staff && \
+addgroup marvin sudo && \
+true
+
##############################################################
-# INSTALL ENGINE FROM SOURCE #
+# INSTALL ENGINE FROM SOURCE #
##############################################################
-COPY {{cookiecutter.project_package}}-0.0.1.tar.gz .
-RUN pip install {{cookiecutter.project_package}}-0.0.1.tar.gz
+ADD {{cookiecutter.project_package}}-0.0.1.tar.gz $MARVIN_HOME
-##############################################################
-# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
-##############################################################
+RUN cd $MARVIN_HOME && \
+mv {{cookiecutter.project_name}} engine && \
+cd $MARVIN_ENGINE_HOME && \
+pip3 install -e . --no-compile
+
+###############################################################
+# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+###############################################################
-RUN pip install --no-cache pandas \
-&& pip install --no-cache matplotlib \
-&& pip install --no-cache cython \
-&& pip install --no-cache numpy \
-&& pip install --no-cache scikit-learn \
-&& pip install --no-cache pytest \
-&& pip install --no-cache pytest-cov \
-&& pip install --no-cache pytest-watch
+RUN pip3 install --no-cache pandas \
+&& pip3 install --no-cache matplotlib \
+&& pip3 install --no-cache cython \
+&& pip3 install --no-cache numpy \
+&& pip3 install --no-cache scikit-learn \
+&& pip3 install --no-cache pytest \
+&& pip3 install --no-cache pytest-cov \
+&& pip3 install --no-cache pytest-watch
##############################################################
# Uninstalling unnecessary software and cleaning cache
@@ -77,6 +108,7 @@ RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
##############################################################
EXPOSE 50051-50057
+EXPOSE 22
EXPOSE 8888
-CMD /bin/bash -c "cd $MARVIN_ENGINE_HOME && marvin-daemon"
\ No newline at end of file
+CMD /bin/bash -c "/usr/sbin/sshd && cd $MARVIN_ENGINE_HOME && nohup marvin-daemon"
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/test.py b/marvin-cli/marvin_cli/management/test.py
old mode 100644
new mode 100755
index 8d3cded..c0a7224
--- a/marvin-cli/marvin_cli/management/test.py
+++ b/marvin-cli/marvin_cli/management/test.py
@@ -23,32 +23,44 @@ def cli():
pass
@cli.command('test', help='Run tests.')
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--cov/--no-cov', default=True)
@click.option('--no-capture', is_flag=True)
@click.option('--pdb', is_flag=True)
@click.argument('args', default='')
-def test(grpchost, grpcport, cov, no_capture, pdb, args):
+@click.pass_context
+def test(ctx, grpchost, grpcport, cov, no_capture, pdb, args):
+ if not grpchost:
+ grpchost = ctx.obj['default_host']
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_test(cov, no_capture, pdb, args)
@cli.command('test-tox', help='Run tests using Tox environment.')
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.argument('args', default='--current-env')
-def tox(grpchost, grpcport, args):
+@click.pass_context
+def tox(ctx, grpchost, grpcport, args):
+ if not grpchost:
+ grpchost = ctx.obj['default_host']
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_tox(args)
@cli.command('test-tdd', help='Watch for changes to run tests automatically.')
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--cov/--no-cov', default=False)
@click.option('--no-capture', is_flag=True)
@click.option('--pdb', is_flag=True)
@click.option('--partial', is_flag=True)
@click.argument('args', default='')
-def tdd(grpchost, grpcport, cov, no_capture, pdb, partial, args):
+@click.pass_context
+def tdd(ctx, grpchost, grpcport, cov, no_capture, pdb, partial, args):
+ if not grpchost:
+ grpchost = ctx.obj['default_host']
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_tdd(cov, no_capture, pdb, partial, args)
diff --git a/marvin-cli/marvin_cli/utils/__init__.py b/marvin-cli/marvin_cli/utils/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/utils/api.py b/marvin-cli/marvin_cli/utils/api.py
new file mode 100644
index 0000000..d352bb0
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/api.py
@@ -0,0 +1,72 @@
+import json
+import time
+import requests
+import sys
+from .log import get_logger
+
+logger = get_logger('utils.api')
+
+class ApiError(Exception):
+ pass
+
+def run_action(host, action_name):
+ resp = requests.post("{0}/{1}".format(host, action_name), json={}, verify=False)
+ if resp.status_code != 200:
+ raise ApiError('POST /{0}/ {1}'.format(action_name, resp.status_code))
+ return resp.json()
+
+def wait_action(host, action_name, protocol):
+ retries = 3
+ while True:
+ payload = {
+ 'protocol': protocol
+ }
+ resp = requests.get("{0}/{1}/status".format(host, action_name),
+ verify=False, params=payload)
+ if resp.status_code != 200:
+ retries -= 1
+ if retries == 0:
+ raise ApiError('PUSH /{0}/status {1}'.format(action_name, resp.status_code))
+ continue
+ retries = 3
+ resp_dict = json.loads(resp.json()['result'])
+ if resp_dict['status']['name'] == 'finished':
+ break
+ else:
+ time.sleep(1)
+
+def run_acquisitor(host, wait=False):
+ try:
+ result = run_action(host, 'acquisitor')
+ if wait:
+ wait_action(host, 'acquisitor', result['result'])
+ except ApiError as e:
+ logger.error(e)
+ sys.exit(1)
+
+def run_tpreparator(host, wait=False):
+ try:
+ result = run_action(host, 'tpreparator')
+ if wait:
+ wait_action(host, 'tpreparator', result['result'])
+ except ApiError as e:
+ logger.error(e)
+ sys.exit(1)
+
+def run_trainer(host, wait=False):
+ try:
+ result = run_action(host, 'trainer')
+ if wait:
+ wait_action(host, 'trainer', result['result'])
+ except ApiError:
+ logger.error(e)
+ sys.exit(1)
+
+def run_evaluator(host, wait=False):
+ try:
+ result = run_action(host, 'evaluator')
+ if wait:
+ wait_action(host, 'evaluator', result['result'])
+ except ApiError:
+ logger.error(e)
+ sys.exit(1)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/benchmark.py b/marvin-cli/marvin_cli/utils/benchmark.py
old mode 100644
new mode 100755
index cff69d3..8c998c5
--- a/marvin-cli/marvin_cli/utils/benchmark.py
+++ b/marvin-cli/marvin_cli/utils/benchmark.py
@@ -31,7 +31,7 @@ logger = get_logger("benchmark")
#to-do
def create_or_get_benchmark_folder():
- _folder_path = os.path.join(os.getcwd(), 'benchmarks')
+ _folder_path = os.path.join(os.environ['MARVIN_DATA_PATH'], 'benchmarks')
if not os.path.exists(_folder_path):
os.makedirs(_folder_path)
return _folder_path
@@ -107,8 +107,8 @@ def filter_data(input_data, initial_time):
int(_r_net),
int(_t_net))
-def get_and_persist_stats(engine_package, initial_time, timestamp):
- _stats = get_stats(engine_package)
+def get_and_persist_stats(engine_name, initial_time, timestamp):
+ _stats = get_stats(engine_name)
_colleted_stats = filter_data(_stats, initial_time)
_filename = 'benchmark_{0}.csv'.format(timestamp)
_path = os.path.join(create_or_get_benchmark_folder(), _filename)
@@ -116,14 +116,14 @@ def get_and_persist_stats(engine_package, initial_time, timestamp):
writer = csv.writer(f)
writer.writerow(_colleted_stats)
-def repeat_stats_call(engine_package, timestamp, initial_time):
+def repeat_stats_call(engine_name, timestamp, initial_time):
while True:
getattr(sys.modules[__name__],
- 'get_and_persist_stats')(engine_package, initial_time, timestamp)
+ 'get_and_persist_stats')(engine_name, initial_time, timestamp)
-def benchmark_thread(engine_package, timestamp, initial_time=time.time()):
+def benchmark_thread(engine_name, timestamp, initial_time=time.time()):
return multiprocessing.Process(target=repeat_stats_call,
- args=(engine_package, timestamp, initial_time,))
+ args=(engine_name, timestamp, initial_time,))
def create_poi(key, value, timestamp):
_file_path = create_or_return_poi(timestamp)
diff --git a/marvin-cli/marvin_cli/utils/config.py b/marvin-cli/marvin_cli/utils/config.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/utils/docker.py b/marvin-cli/marvin_cli/utils/docker.py
old mode 100644
new mode 100755
index 5a605b7..6dfaa62
--- a/marvin-cli/marvin_cli/utils/docker.py
+++ b/marvin-cli/marvin_cli/utils/docker.py
@@ -18,7 +18,9 @@
import os
from docker import DockerClient
from docker.errors import ImageNotFound, NotFound
-from .misc import package_to_name, generate_engine_package
+from shutil import copyfile
+from .misc import package_to_name, generate_engine_package, name_to_package
+from .misc import get_chunk_and_untar, get_tar_data
from ..utils.log import get_logger
logger = get_logger('docker')
@@ -28,6 +30,9 @@ def _get_client(env=True, url=None):
return DockerClient.from_env()
return DockerClient(base_url=url)
+def rename_image(old_name, new_name):
+ os.system("docker tag {} {}".format(old_name, new_name))
+
def _search_docker_image(name):
client = _get_client()
try:
@@ -44,48 +49,108 @@ def _search_docker_container(name):
except NotFound:
return False
-def search_engine_container(engine_package):
- container_name = "marvin-cont-" + package_to_name(engine_package)
+def search_docker_volume(name):
+ client = _get_client()
+ try:
+ client.volumes.get(name)
+ return True
+ except NotFound:
+ return False
+
+def search_engine_container(engine_name):
+ container_name = "marvin-cont-" + engine_name
return _search_docker_container(container_name)
-def search_engine_images(engine_package):
- image_name = "marvin-" + package_to_name(engine_package)
+def search_engine_images(engine_name):
+ image_name = "marvin-" + engine_name
return _search_docker_image(image_name)
-def create_engine_image(engine_package):
+def create_engine_image(engine_name, engine_path):
logger.info("Creating engine docker image ...")
- dockerfile_path = os.path.join(os.getcwd(), "docker",
+ _package = name_to_package(engine_name)
+ dockerfile_path = os.path.join(engine_path, "docker",
"develop", "daemon")
- generate_engine_package(engine_package)
+ generate_engine_package(_package, engine_path)
client = _get_client()
client.images.build(
path=dockerfile_path,
- tag="marvin-" + package_to_name(engine_package)
+ tag="marvin-" + engine_name
)
logger.info("Creating engine docker image ... Done!")
-def create_daemon_container(engine_package, engine_name):
+def create_deploy_image_and_push(engine_name, engine_path, name_in_registry):
+ logger.info("Creating engine deploy image ...")
+ _package = name_to_package(engine_name)
+ dockerfile_path = os.path.join(engine_path, "docker",
+ "deploy", "daemon")
+ generate_engine_package(_package, engine_path,
+ dest=dockerfile_path)
+
+ client = _get_client()
+ client.images.build(
+ path=dockerfile_path,
+ tag=name_in_registry
+ )
+ logger.info("Creating engine deploy image... Done!")
+ logger.info("Pushing deploy image to registry...")
+ os.system("docker push {}".format(name_in_registry))
+ logger.info("Pushing deploy image to registry... Done!")
+
+def create_docker_volume(volume_name):
+ client = _get_client()
+ logger.info("Creating {} docker volume...".format(volume_name))
+ client.volumes.create(volume_name)
+ logger.info("Creating {} docker volume... Done!".format(volume_name))
+
+def shutdown_and_delete_container(name):
+ client = _get_client()
+ try:
+ _container = client.containers.get(name)
+ _container.stop()
+ logger.warning("{} container was stopped!".format(name))
+ _container.remove()
+ logger.info("{} container was removed!".format(name))
+ except NotFound:
+ logger.warning("Container not found!")
+
+def delete_image_and_volume(name, vol):
+ client = _get_client()
+ try:
+ client.images.remove(name)
+ logger.warning("{} image was untagged, use prune to delete it permanently!".format(name))
+ _vol = client.volumes.get(vol)
+ _vol.remove()
+ logger.warning("{} volume was deleted!".format(vol))
+ except NotFound:
+ logger.warning("Image or volume not found!")
+
+def create_daemon_container(engine_name):
logger.info("Creating engine docker container ...")
+
client = _get_client()
- _engine_path = os.path.join(os.environ['MARVIN_HOME'], engine_name)
+ _engine_volume = "marvin-{}-vol".format(engine_name)
+ _data_volume = "marvin-data"
+ _log_volume = "marvin-log"
+
client.containers.run(
image="marvin-" + engine_name,
name="marvin-cont-" + engine_name,
volumes={
- _engine_path: {
- 'bind': '/opt/marvin/engine',
+ _engine_volume: {
+ 'bind': '/home/marvin/engine',
'mode': 'rw'
},
- os.environ['MARVIN_DATA_PATH']: {
- 'bind': '/opt/marvin/data',
+ _data_volume: {
+ 'bind': '/home/marvin/data',
'mode': 'rw'
},
- os.environ['MARVIN_LOG']: {
- 'bind': '/opt/marvin/log',
+ _log_volume: {
+ 'bind': '/home/marvin/log',
'mode': 'rw'
}
},
ports={
+ '22/tcp':2022,
'50051/tcp': 50051,
'50052/tcp': 50052,
'50053/tcp': 50053,
@@ -100,12 +165,165 @@ def create_daemon_container(engine_package, engine_name):
logger.info("Creating engine docker container ... Done!")
-def get_stats(engine_package):
+def create_executor_container(engine_name):
+ logger.info("Creating engine executor container ...")
+
+ client = _get_client()
+ _engine_volume = "marvin-{}-vol".format(engine_name)
+
+ client.containers.run(
+ image="marvin-executor",
+ name="marvin-executor-" + engine_name,
+ volumes={
+ _engine_volume: {
+ 'bind': '/home/marvin/engine',
+ 'mode': 'rw'
+ }
+ },
+ ports={
+ '8000/tcp': 8000
+ },
+ detach=True
+ )
+
+ logger.info("Creating executor docker container ... Done!")
+
+def create_tfserving_container(engine_name, model_path, serve=False):
+ logger.info("Creating temporary container ...")
+
+ _container_name = "tfserving-{}".format(engine_name)
+ _dest_path = "/models/{}".format(engine_name)
+
+ client = _get_client()
+ container = client.containers.run(
+ image="tensorflow/serving:1.11.1",
+ name=_container_name,
+ detach=True
+ )
+ logger.info("Creating temporary container ... Done!")
+
+ logger.info("Writing model on image ...")
+ os.system("docker cp {0} {1}:/models/{2}".format(model_path, _container_name, engine_name))
+
+ os.system(
+ "docker commit --change \'ENV MODEL_NAME {0}\' {1} tfserving:{0}"
+ .format(engine_name, _container_name)
+ )
+ logger.info("Writing model on image ... Done!")
+
+ logger.info("Cleaning...")
+ shutdown_and_delete_container(_container_name)
+ logger.info("Cleaning... Done!")
+
+ if serve:
+ logger.info("Serving...")
+ client.containers.run(
+ image="tensorflow/serving:{}".format(engine_name),
+ name=_container_name,
+ ports={
+ '8500/tcp': 8500
+ },
+ detach=True
+ )
+ logger.info("Serving... Done!")
+
+
+def get_stats(engine_name):
client = _get_client()
- if search_engine_container(engine_package):
- container_name = "marvin-cont-" + package_to_name(engine_package)
+ if search_engine_container(engine_name):
+ container_name = "marvin-cont-" + engine_name
container = client.containers.get(container_name)
return container.stats(stream=False)
else:
logger.error("Engine container was not found!")
- return None
\ No newline at end of file
+ return None
+
+class DaemonManagement:
+
+ data_path = '/home/marvin/data'
+ log_path = '/home/marvin/log'
+
+ def __init__(self, engine_name):
+ self.engine_name = engine_name
+ self.engine_path = '/home/marvin/engine'
+
+ @staticmethod
+ def get_container(engine_name):
+ _engine_container = "marvin-cont-{}".format(engine_name)
+ client = _get_client()
+ return client.containers.get(_engine_container)
+
+ def delete_from_daemon(self, source, folder):
+ if folder:
+ source = os.path.join(source, '*')
+
+ _container = self.get_container(self.engine_name)
+ _response = _container.exec_run("sh -c 'rm -rf {}'".format(source))
+
+ if _response.exit_code == 0:
+ logger.info("Files were deleted sucessfully!")
+ else:
+ logger.error("Error deleting files.")
+
+ def list_dir_from_daemon(self, source):
+ _container = self.get_container(self.engine_name)
+
+ _response = _container.exec_run("ls {}".format(source))
+ if _response.exit_code == 0:
+ print(_response.output.decode())
+ else:
+ logger.error("Error listing files.")
+
+ def copy_from_daemon(self, source, dest):
+ _container = self.get_container(self.engine_name)
+ bits, stats = _container.get_archive(source)
+ logger.info("Copying file from daemon container. Size:{} bytes...".format(stats['size']))
+ get_chunk_and_untar(bits, dest)
+ logger.info("Copying file from daemon container. Size:{} bytes... Done!".format(stats['size']))
+
+ def copy_to_daemon(self, source, dest, folder=True):
+ _container = self.get_container(self.engine_name)
+ data, filepath = get_tar_data(source, folder)
+
+ _response = _container.put_archive(dest, data)
+
+ if _response:
+ logger.info("File copy was sucessfull!")
+ else:
+ logger.error("Error in file copy.")
+ #delete tmp file
+ os.remove(filepath)
+
+ def push_data(self, path):
+ self.copy_to_daemon(path, self.data_path, compress, folder=False)
+
+ def delete_data(self, file):
+ _file_path = os.path.join(self.data_path, file)
+ self.delete_from_daemon(_file_path, folder=False)
+
+ def list_data_files(self):
+ self.list_dir_from_daemon(self.data_path)
+
+ def clone_engine(self):
+ _dest = os.environ['MARVIN_HOME']
+ _orig_folder_name = os.path.join(_dest, 'engine')
+ _mod_folder_name = os.path.join(_dest, self.engine_name)
+ self.copy_from_daemon(self.engine_path, _dest)
+ os.rename(_orig_folder_name, _mod_folder_name)
+
+ def push_engine(self):
+ _source = os.environ['MARVIN_HOME']
+ _mod_folder_name = os.path.join(_source, self.engine_name)
+ _engine_folder = os.path.join(_source, 'engine')
+ os.rename(_mod_folder_name, _engine_folder)
+ self.delete_from_daemon(self.engine_path, folder=True)
+ self.copy_to_daemon(_engine_folder, self.engine_path)
+
+ def clone_artifacts(self):
+ _dest = os.path.join(os.environ['MARVIN_DATA_PATH'], '.artifacts', self.engine_name)
+ _source = os.path.join(self.data_path, '.artifacts', self.engine_name)
+ self.copy_from_daemon(_source, _dest)
+
+ def clone_logs(self):
+ _dest = os.path.join(os.environ['MARVIN_DATA_PATH'], '.log', self.engine_name)
+ self.copy_from_daemon(self.log_path, _dest)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/git.py b/marvin-cli/marvin_cli/utils/git.py
old mode 100644
new mode 100755
index 6dec396..8eaff3b
--- a/marvin-cli/marvin_cli/utils/git.py
+++ b/marvin-cli/marvin_cli/utils/git.py
@@ -21,8 +21,8 @@ def git_init(path):
os.chdir(path)
os.system(' '.join(['git', 'init', '.']))
-def bump_version(part, verbose, dryrun):
- command = ['bump2version', part]
+def bump_version(path, part, verbose, dryrun):
+ command = ['bump2version', part, path]
if verbose:
command.append('--verbose')
if dryrun:
diff --git a/marvin-cli/marvin_cli/utils/log.py b/marvin-cli/marvin_cli/utils/log.py
old mode 100644
new mode 100755
index 2417977..a8a00a0
--- a/marvin-cli/marvin_cli/utils/log.py
+++ b/marvin-cli/marvin_cli/utils/log.py
@@ -26,12 +26,6 @@ import logging
DEFAULT_LOG_LEVEL = logging.INFO
-if os.environ.get('MARVIN_LOG') is None:
- DEFAULT_LOG_DIR = '/tmp'
-else:
- DEFAULT_LOG_DIR = os.environ['MARVIN_LOG']
-
-
class Logger(logging.getLoggerClass()):
"""Custom logger class.
@@ -51,13 +45,11 @@ logging.setLoggerClass(Logger)
def get_logger(name, namespace='marvin_ai',
- log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ log_level=DEFAULT_LOG_LEVEL):
"""Build a logger that outputs to a file and to the console,"""
log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
os.getenv('LOG_LEVEL', log_level))
- log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
- os.getenv('LOG_DIR', log_dir))
logger = logging.getLogger('{}.{}'.format(namespace, name))
logger.setLevel(log_level)
@@ -71,23 +63,4 @@ def get_logger(name, namespace='marvin_ai',
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
- try:
- log_path = os.path.abspath(log_dir)
- log_filename = '{name}.{pid}.log'.format(
- name=namespace, pid=os.getpid())
-
- file_path = str(os.path.join(log_path, log_filename))
-
- if not os.path.exists(log_path):
- os.makedirs(log_path)
-
- # Create a file handler
- file_handler = logging.FileHandler(file_path)
- file_handler.setLevel(log_level)
- file_handler.setFormatter(formatter)
- logger.addHandler(file_handler)
- except OSError as e:
- logger.error('Could not create log file {file}: {error}'.format(
- file=file_path, error=e.strerror))
-
return logger
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/misc.py b/marvin-cli/marvin_cli/utils/misc.py
old mode 100644
new mode 100755
index 84d7974..5f4d01b
--- a/marvin-cli/marvin_cli/utils/misc.py
+++ b/marvin-cli/marvin_cli/utils/misc.py
@@ -16,12 +16,18 @@
# limitations under the License.
import os
+import sys
import subprocess
import tarfile
import wget
import glob
import pickle
import datetime
+import time
+import shutil
+from cryptography.hazmat.primitives import serialization as crypto_serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.backends import default_backend as crypto_default_backend
from .log import get_logger
logger = get_logger('misc')
@@ -34,15 +40,23 @@ def package_to_name(package):
#remove marvin_ substring
return package[len("marvin_"):]
-def generate_engine_package(package):
- subprocess.run(['python', 'setup.py', 'sdist'])
- filename = package + "-" + get_version(package) + ".tar.gz"
- origin = os.path.join(os.getcwd(), "dist", filename)
- dest = os.path.join(os.getcwd(), "docker", "develop" ,"daemon", filename)
- os.rename(origin, dest)
+def name_to_package(name):
+ return "marvin_{}".format(name)
-def get_version(package):
- with open(os.path.join(os.getcwd(), package ,"VERSION"), 'rb') as f:
+def generate_engine_package(package, path, dest=None):
+ filename = package + "-" + get_version(package, path) + ".tar.gz"
+ output = os.path.join('/tmp/marvin', filename)
+ make_tarfile(output, path)
+
+ if dest is not None:
+ move_dest = os.path.join(dest, filename)
+ else:
+ move_dest = os.path.join(path, "docker", "develop" ,"daemon", filename)
+
+ shutil.move(output, move_dest)
+
+def get_version(package, path):
+ with open(os.path.join(path, package ,"VERSION"), 'rb') as f:
version = f.read().decode('ascii').strip()
return version
@@ -54,14 +68,9 @@ def extract_folder(input, output):
tf = tarfile.open(input)
tf.extractall(output)
-def call_logs(package, follow, buffer):
- container_name = 'marvin-cont-' + package_to_name(package)
- p_return = None
- if follow:
- p_return = subprocess.Popen(['docker', 'logs', '--follow', container_name], stdout=subprocess.PIPE)
- else:
- p_return = subprocess.Popen(['docker', 'logs', '--tail', str(buffer), container_name], stdout=subprocess.PIPE)
-
+def call_logs(engine):
+ container_name = 'marvin-cont-' + engine
+ p_return = subprocess.Popen(['docker', 'logs', '--follow', container_name], stdout=subprocess.PIPE)
return p_return
def create_or_return_tmp_dir():
@@ -70,27 +79,21 @@ def create_or_return_tmp_dir():
os.makedirs(tmp_path)
return tmp_path
-def kill_persisted_process():
- base_path = create_or_return_tmp_dir()
- for obj_file in glob.glob('{0}/*.mproc'.format(base_path)):
- pid = int(
- obj_file[(len(base_path) + 1):(len('.mproc') * -1)]
- )
- try:
- os.kill(pid, 9)
- logger.info("PID {0} now killed!".format(pid))
- except ProcessLookupError:
- logger.info("PID {0} already killed!".format(pid))
- os.remove(obj_file)
-
-
-def persist_process(obj):
- filepath = os.path.join(create_or_return_tmp_dir(),
- '{0}.mproc'.format(obj.pid))
- logger.info("Creating {0}...".format(filepath))
- with open(filepath, 'w'):
- pass
-
+def write_tmp_info(key, info):
+ _filepath = os.path.join(create_or_return_tmp_dir(), key)
+ logger.info("Creating {0}...".format(key))
+ with open(_filepath, 'w') as f:
+ f.write(info)
+
+def retrieve_tmp_info(key):
+ _filepath = os.path.join(create_or_return_tmp_dir(), key)
+ logger.info("Retriving {0}...".format(key))
+ try:
+ with open(_filepath, 'r') as f:
+ info = f.read()
+ return info
+ except:
+ return None
def get_executor_path_or_download(executor_url):
#get filename from url
@@ -105,4 +108,98 @@ def get_executor_path_or_download(executor_url):
return executor_path
def generate_timestamp():
- return datetime.datetime.now().timestamp()
\ No newline at end of file
+ return datetime.datetime.now().timestamp()
+
+def create_tmp_marvin_folder():
+ _dir = '/tmp/marvin'
+ if not os.path.exists(_dir):
+ os.makedirs(_dir)
+
+def get_chunk_and_untar(bits, output_path):
+ _dir = '/tmp/marvin'
+ _tmp_path = os.path.join(_dir, 'tmp_data')
+ #save tar in tmp file
+ with open(_tmp_path, 'wb') as f:
+ for chunk in bits:
+ f.write(chunk)
+ f.close()
+ #extract files
+ with tarfile.open(_tmp_path) as tf:
+ tf.extractall(output_path)
+ tf.close()
+ #remove tmp_data
+ os.remove(_tmp_path)
+
+def get_tar_data(source, folder, compress):
+ _dir = '/tmp/marvin'
+ tmp_path = os.path.join(_dir, 'tmp_data')
+ _tar_mode = "w:gz" if compress else "w"
+ #save tar in tmp file
+ with tarfile.open(tmp_path, _tar_mode) as tf:
+ if folder:
+ tf.add(source, arcname='.')
+ else:
+ tf.add(source, arcname=os.path.basename(source))
+
+ #get bytes from file
+ with open(tmp_path, 'rb') as bf:
+ temp_bytes = bf.read()
+ return (temp_bytes, tmp_path)
+
+def generate_keys(engine_name):
+ _key_path = os.path.join(os.environ['MARVIN_DATA_PATH'],
+ '.keys',
+ engine_name)
+
+ os.makedirs(_key_path)
+
+ pvk_path = os.path.join(_key_path, 'id_rsa')
+ pubk_path = os.path.join(_key_path, 'id_rsa.pub')
+
+ key = rsa.generate_private_key(
+ backend=crypto_default_backend(),
+ public_exponent=65537,
+ key_size=2048
+ )
+
+ private_key = key.private_bytes(
+ crypto_serialization.Encoding.PEM,
+ crypto_serialization.PrivateFormat.PKCS8,
+ crypto_serialization.NoEncryption()
+ )
+
+ public_key = key.public_key().public_bytes(
+ crypto_serialization.Encoding.OpenSSH,
+ crypto_serialization.PublicFormat.OpenSSH
+ )
+
+ open(pubk_path ,"w").write(public_key.decode("utf-8"))
+ open(pvk_path ,"w").write(private_key.decode("utf-8"))
+ os.chmod(pvk_path, 0o500)
+
+ return pubk_path
+
+def init_port_forwarding(engine_name, remote_host, ports_list, background=True):
+ if remote_host != 'localhost' and remote_host != '127.0.0.1':
+ pkey_path = os.path.join(os.environ['MARVIN_DATA_PATH'], '.keys', engine_name, 'id_rsa')
+ command_list = ["ssh"]
+ command_list.append("-o")
+ command_list.append("StrictHostKeyChecking=no")
+ command_list.append("-N")
+
+ if background:
+ command_list.append('-f')
+
+ for remote_port in ports_list:
+ command_list.append("-L")
+ command_list.append("localhost:{0}:localhost:{0}".format(remote_port))
+
+ command_list.append("-i")
+ command_list.append("{0}".format(pkey_path))
+ command_list.append("marvin@{0}".format(remote_host))
+ command_list.append("-p")
+ command_list.append("2022")
+ if not background:
+ logger.info("Press Ctrl+C to disable port forwarding")
+
+ os.system(" ".join(command_list))
\ No newline at end of file
diff --git a/marvin-cli/pytest.ini b/marvin-cli/pytest.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/setup.py b/marvin-cli/setup.py
old mode 100644
new mode 100755
index 82b3eba..5102069
--- a/marvin-cli/setup.py
+++ b/marvin-cli/setup.py
@@ -55,13 +55,18 @@ REQUIREMENTS_EXTERNAL = [
'click',
'cookiecutter',
'docker-compose',
- 'matplotlib'
+ 'matplotlib',
+ 'cryptography',
+ 'kubernetes',
+ 'requests'
]
# This is normally an empty list
DEPENDENCY_LINKS_EXTERNAL = []
# script to be used
-SCRIPTS = ['bin/marvin', 'bin/marvin_complete']
+SCRIPTS = ['bin/marvin',
+ 'bin/marvin_complete',
+ 'bin/marvin-api']
def _get_version():
diff --git a/marvin-cli/tests/communication/test_remote_calls.py b/marvin-cli/tests/communication/test_remote_calls.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/management/test_edit.py b/marvin-cli/tests/management/test_edit.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/management/test_engine.py b/marvin-cli/tests/management/test_engine.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_benchmark.py b/marvin-cli/tests/utils/test_benchmark.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_config.py b/marvin-cli/tests/utils/test_config.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_docker.py b/marvin-cli/tests/utils/test_docker.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_git.py b/marvin-cli/tests/utils/test_git.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_misc.py b/marvin-cli/tests/utils/test_misc.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tox.ini b/marvin-cli/tox.ini
old mode 100644
new mode 100755