You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marvin.apache.org by we...@apache.org on 2021/10/23 13:13:15 UTC
[incubator-marvin] branch develop updated (2ac8a7f -> 0fa22ce)
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git.
from 2ac8a7f Allowing the predictor endpoint to return JSON within result, in addition to strings
new f5f9639 Quick fixes at CLI and Airflow/Kubernetes templates
new 212a84b Fix .travis.yml
new 5b89321 Fix CLI tests tensorflow serializer version on tox
new 0fa22ce Fix benchmark test
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 8 +-
marvin-cli/MANIFEST.in | 2 +
marvin-cli/Makefile | 0
marvin-cli/README.md | 0
marvin-cli/bin/marvin | 59 ++--
marvin-cli/bin/marvin-api | 50 +++
marvin-cli/bin/marvin_complete | 0
marvin-cli/marvin_cli/VERSION | 0
marvin-cli/marvin_cli/__init__.py | 0
marvin-cli/marvin_cli/communication/__init__.py | 0
.../marvin_cli/communication/remote_calls.py | 18 +-
.../marvin_cli/communication/stubs/__init__.py | 0
.../marvin_cli/communication/stubs/daemon_pb2.py | 0
.../communication/stubs/daemon_pb2_grpc.py | 0
marvin-cli/marvin_cli/management/__init__.py | 0
.../management/airflow_template/cookiecutter.json | 8 +
.../{{cookiecutter.dag_name}}.py | 53 +++
marvin-cli/marvin_cli/management/edit.py | 6 -
marvin-cli/marvin_cli/management/engine.py | 379 ++++++++++++++++-----
marvin-cli/marvin_cli/management/generate.py | 56 +++
.../kubernetes_template/cookiecutter.json | 9 +
.../{{cookiecutter.engine_name}}/deploy.sh | 6 +
.../{{cookiecutter.engine_name}}_deployment.yaml | 21 ++
.../{{cookiecutter.engine_name}}_service.yaml | 11 +
marvin-cli/marvin_cli/management/notebook.py | 34 +-
.../management/template/cookiecutter.json | 0
.../{{cookiecutter.project_name}}/.bumpversion.cfg | 0
.../{{cookiecutter.project_name}}/.coveragerc | 0
.../{{cookiecutter.project_name}}/.gitignore | 0
.../{{cookiecutter.project_name}}/CHANGES.md | 0
.../template/{{cookiecutter.project_name}}/INSTALL | 0
.../template/{{cookiecutter.project_name}}/LICENSE | 0
.../{{cookiecutter.project_name}}/MANIFEST.in | 3 +-
.../{{cookiecutter.project_name}}/Makefile | 0
.../{{cookiecutter.project_name}}/README.md | 0
.../docker/{develop => deploy}/daemon/Dockerfile | 76 +++--
.../docker/deploy/daemon/deps/start.sh | 2 +
.../docker/develop/daemon/Dockerfile | 76 +++--
.../{{cookiecutter.project_name}}/docs.yaml | 0
.../{{cookiecutter.project_name}}/engine.messages | 0
.../{{cookiecutter.project_name}}/engine.metadata | 0
.../{{cookiecutter.project_name}}/engine.params | 0
.../feedback.messages | 0
.../{{cookiecutter.project_name}}/marvin.ini | 0
.../notebooks/sample.ipynb | 0
.../{{cookiecutter.project_name}}/pytest.ini | 0
.../scripts/install_automl.sh | 0
.../{{cookiecutter.project_name}}/setup.cfg | 0
.../{{cookiecutter.project_name}}/setup.py | 0
.../tests/conftest.py | 0
.../data_handler/test_acquisitor_and_cleaner.py | 0
.../tests/data_handler/test_training_preparator.py | 0
.../tests/prediction/test_feedback.py | 0
.../tests/prediction/test_prediction_preparator.py | 0
.../tests/prediction/test_predictor.py | 0
.../tests/training/test_metrics_evaluator.py | 0
.../tests/training/test_trainer.py | 0
.../template/{{cookiecutter.project_name}}/tox.ini | 0
.../{{cookiecutter.project_package}}/VERSION | 0
.../{{cookiecutter.project_package}}/__init__.py | 0
.../_compatibility.py | 0
.../{{cookiecutter.project_package}}/_logging.py | 0
.../data_handler/__init__.py | 0
.../data_handler/acquisitor_and_cleaner.py | 0
.../data_handler/training_preparator.py | 0
.../prediction/__init__.py | 0
.../prediction/feedback.py | 0
.../prediction/prediction_preparator.py | 0
.../prediction/predictor.py | 0
.../training/__init__.py | 0
.../training/metrics_evaluator.py | 0
.../training/trainer.py | 0
marvin-cli/marvin_cli/management/test.py | 30 +-
marvin-cli/marvin_cli/utils/__init__.py | 0
marvin-cli/marvin_cli/utils/api.py | 72 ++++
marvin-cli/marvin_cli/utils/benchmark.py | 14 +-
marvin-cli/marvin_cli/utils/config.py | 0
marvin-cli/marvin_cli/utils/docker.py | 260 ++++++++++++--
marvin-cli/marvin_cli/utils/git.py | 4 +-
marvin-cli/marvin_cli/utils/log.py | 29 +-
marvin-cli/marvin_cli/utils/misc.py | 173 +++++++---
marvin-cli/pytest.ini | 0
marvin-cli/setup.py | 9 +-
.../tests/communication/test_remote_calls.py | 0
marvin-cli/tests/management/test_edit.py | 7 -
marvin-cli/tests/management/test_engine.py | 0
marvin-cli/tests/utils/test_benchmark.py | 2 +-
marvin-cli/tests/utils/test_config.py | 0
marvin-cli/tests/utils/test_docker.py | 13 +-
marvin-cli/tests/utils/test_git.py | 4 +-
marvin-cli/tests/utils/test_misc.py | 10 +-
marvin-cli/tox.ini | 0
python-daemon/tox.ini | 1 +
93 files changed, 1181 insertions(+), 324 deletions(-)
mode change 100644 => 100755 marvin-cli/MANIFEST.in
mode change 100644 => 100755 marvin-cli/Makefile
mode change 100644 => 100755 marvin-cli/README.md
mode change 100644 => 100755 marvin-cli/bin/marvin
create mode 100644 marvin-cli/bin/marvin-api
mode change 100644 => 100755 marvin-cli/bin/marvin_complete
mode change 100644 => 100755 marvin-cli/marvin_cli/VERSION
mode change 100644 => 100755 marvin-cli/marvin_cli/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/communication/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/communication/remote_calls.py
mode change 100644 => 100755 marvin-cli/marvin_cli/communication/stubs/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py
mode change 100644 => 100755 marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/__init__.py
create mode 100644 marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json
create mode 100644 marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/edit.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/engine.py
create mode 100644 marvin-cli/marvin_cli/management/generate.py
create mode 100755 marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json
create mode 100644 marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh
create mode 100644 marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml
create mode 100644 marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml
mode change 100644 => 100755 marvin-cli/marvin_cli/management/notebook.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/cookiecutter.json
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md
copy marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/{develop => deploy}/daemon/Dockerfile (54%)
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py
mode change 100644 => 100755 marvin-cli/marvin_cli/management/test.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/__init__.py
create mode 100644 marvin-cli/marvin_cli/utils/api.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/benchmark.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/config.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/docker.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/git.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/log.py
mode change 100644 => 100755 marvin-cli/marvin_cli/utils/misc.py
mode change 100644 => 100755 marvin-cli/pytest.ini
mode change 100644 => 100755 marvin-cli/setup.py
mode change 100644 => 100755 marvin-cli/tests/communication/test_remote_calls.py
mode change 100644 => 100755 marvin-cli/tests/management/test_edit.py
mode change 100644 => 100755 marvin-cli/tests/management/test_engine.py
mode change 100644 => 100755 marvin-cli/tests/utils/test_benchmark.py
mode change 100644 => 100755 marvin-cli/tests/utils/test_config.py
mode change 100644 => 100755 marvin-cli/tests/utils/test_docker.py
mode change 100644 => 100755 marvin-cli/tests/utils/test_git.py
mode change 100644 => 100755 marvin-cli/tests/utils/test_misc.py
mode change 100644 => 100755 marvin-cli/tox.ini
[incubator-marvin] 03/04: Fix CLI tests tensorflow serializer
version on tox
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit 5b89321476ed205010296a3de0e0268759b458a9
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Thu Oct 21 21:46:45 2021 -0300
Fix CLI tests tensorflow serializer version on tox
---
.travis.yml | 5 ++---
marvin-cli/tests/management/test_edit.py | 7 -------
marvin-cli/tests/utils/test_benchmark.py | 2 +-
marvin-cli/tests/utils/test_docker.py | 13 +++++++------
marvin-cli/tests/utils/test_git.py | 4 ++--
marvin-cli/tests/utils/test_misc.py | 10 +++++-----
python-daemon/tox.ini | 1 +
7 files changed, 18 insertions(+), 24 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 9ead5a1..525b246 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,15 +50,14 @@ matrix:
- export MARVIN_DATA_PATH=./marvin_data
- export SPARK_HOME=../spark-2.1.1-bin-hadoop2.6
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libsasl2-dev python-pip graphviz -y ; fi
- - travis_retry sudo pip install --upgrade pip
- - travis_retry sudo pip install virtualenvwrapper --ignore-installed six
+ - travis_retry pip install --upgrade pip
+ - travis_retry pip install virtualenvwrapper --ignore-installed six
- source virtualenvwrapper.sh
install:
- travis_retry pip install codecov
- travis_retry pip install unidecode
- travis_retry pip install tox
- make install
- - travis_retry pip install 'h5py==2.10.0' --force-reinstall
script:
- tox
- codecov
diff --git a/marvin-cli/tests/management/test_edit.py b/marvin-cli/tests/management/test_edit.py
index 00eba18..3b1b630 100755
--- a/marvin-cli/tests/management/test_edit.py
+++ b/marvin-cli/tests/management/test_edit.py
@@ -23,7 +23,6 @@ except ImportError:
import unittest.mock as mock
from marvin_cli.management.edit import config
-from marvin_cli.management.edit import metadata
mocked_obj = {
'editor': 'mocked'
}
@@ -34,11 +33,5 @@ def test_config():
runner = click.testing.CliRunner()
runner.invoke(config)
-def test_metadata():
- ctx = click.Context(click.Command('edit-metadata'), obj=mocked_obj)
- with ctx:
- runner = click.testing.CliRunner()
- runner.invoke(metadata)
-
diff --git a/marvin-cli/tests/utils/test_benchmark.py b/marvin-cli/tests/utils/test_benchmark.py
index 2162779..d011f72 100755
--- a/marvin-cli/tests/utils/test_benchmark.py
+++ b/marvin-cli/tests/utils/test_benchmark.py
@@ -68,7 +68,7 @@ mocked_data = {
def test_create_or_get_benchmark_folder():
path = create_or_get_benchmark_folder()
- assert path == os.path.join(os.getcwd(), 'benchmarks')
+ assert path == os.path.join(os.environ['MARVIN_HOME'], 'benchmarks')
def test_create_or_return_poi():
timestamp = 'mocked_timestamp'
diff --git a/marvin-cli/tests/utils/test_docker.py b/marvin-cli/tests/utils/test_docker.py
index 5360883..25167d4 100755
--- a/marvin-cli/tests/utils/test_docker.py
+++ b/marvin-cli/tests/utils/test_docker.py
@@ -47,21 +47,22 @@ def test_search_docker_container():
@mock.patch('marvin_cli.utils.docker._search_docker_container')
def test_search_engine_container(search_mocked):
search_engine_container('marvin_mocked')
- search_mocked.assert_called_with('marvin-cont-mocked')
+ search_mocked.assert_called_with('marvin-cont-marvin_mocked')
@mock.patch('marvin_cli.utils.docker._search_docker_image')
def test_search_engine_images(search_mocked):
search_engine_images('marvin_mocked')
- search_mocked.assert_called_with('marvin-mocked')
+ search_mocked.assert_called_with('marvin-marvin_mocked')
@mock.patch('marvin_cli.utils.docker._get_client')
@mock.patch('marvin_cli.utils.docker.generate_engine_package')
def test_create_engine_image(generate_mocked, client_mocked):
- create_engine_image('marvin_mocked')
+ mocked_engine_path = 'mocked/path/'
+ create_engine_image('marvin_mocked', mocked_engine_path)
client_mocked.assert_called()
- generate_mocked.assert_called_with('marvin_mocked')
+ generate_mocked.assert_called_with('marvin_marvin_mocked', mocked_engine_path)
@mock.patch('marvin_cli.utils.docker._get_client')
def test_create_daemon_container(client_mocked):
- create_daemon_container('marvin_mocked', 'mocked')
- client_mocked.assert_called()
\ No newline at end of file
+ create_daemon_container('marvin_mocked')
+ client_mocked.assert_called()
diff --git a/marvin-cli/tests/utils/test_git.py b/marvin-cli/tests/utils/test_git.py
index a20ea5a..552f6d2 100755
--- a/marvin-cli/tests/utils/test_git.py
+++ b/marvin-cli/tests/utils/test_git.py
@@ -34,5 +34,5 @@ def test_git_init(chdir_mocked, system_mocked):
@mock.patch('marvin_cli.utils.git.os.system')
def test_bumpversion(system_mocked):
- bump_version('patch', True, True)
- system_mocked.assert_called_with('bump2version patch --verbose --dry-run')
\ No newline at end of file
+ bump_version('patch', 'mocked', True, True)
+ system_mocked.assert_called_with('bump2version mocked patch --verbose --dry-run')
diff --git a/marvin-cli/tests/utils/test_misc.py b/marvin-cli/tests/utils/test_misc.py
index b52f188..2a629ef 100755
--- a/marvin-cli/tests/utils/test_misc.py
+++ b/marvin-cli/tests/utils/test_misc.py
@@ -44,8 +44,8 @@ def test_package_to_name():
@mock.patch('marvin_cli.utils.misc.os.path.join')
def test_get_version(join_mocked):
- get_version(mocked_package)
- join_mocked.assert_called_with(os.getcwd(), mocked_package, 'VERSION')
+ get_version(mocked_package, mocked_path)
+ join_mocked.assert_called_with(mocked_path, mocked_package, 'VERSION')
@mock.patch('marvin_cli.utils.misc.tarfile.open')
def test_package_folder(open_mocked):
@@ -59,10 +59,10 @@ def test_extract_folder(open_mocked):
@mock.patch('marvin_cli.utils.misc.subprocess.Popen')
def test_call_logs(popen_mocked):
- call_logs(mocked_package, True, 0)
- popen_mocked.assert_called_with(['docker', 'logs', '--follow', 'marvin-cont-mocked'], stdout=-1)
+ call_logs(mocked_package)
+ popen_mocked.assert_called_with(['docker', 'logs', '--follow', 'marvin-cont-' + mocked_package], stdout=-1)
@mock.patch('marvin_cli.utils.misc.wget.download')
def test_get_executor_path_or_download(wget_mocked):
get_executor_path_or_download(mocked_url)
- wget_mocked.assert_called_with(mocked_url, out=os.path.join(os.environ['MARVIN_DATA_PATH'], 'marvin.jar'))
\ No newline at end of file
+ wget_mocked.assert_called_with(mocked_url, out=os.path.join(os.environ['MARVIN_DATA_PATH'], 'marvin.jar'))
diff --git a/python-daemon/tox.ini b/python-daemon/tox.ini
index 8e918b7..059e2f5 100644
--- a/python-daemon/tox.ini
+++ b/python-daemon/tox.ini
@@ -6,5 +6,6 @@ deps=pytest
pytest-cov
mock
tensorflow
+ h5py==2.10.0
commands=py.test --cov={envsitepackagesdir}/marvin_python_daemon --cov-report html --cov-report xml {posargs}
passenv=SPARK_HOME MARVIN_HOME MARVIN_DATA_PATH MARVIN_LOG
[incubator-marvin] 04/04: Fix benchmark test
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit 0fa22ce1fa8312a515cefc370c7c18ffeb17990e
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Thu Oct 21 21:58:01 2021 -0300
Fix benchmark test
---
marvin-cli/tests/utils/test_benchmark.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/marvin-cli/tests/utils/test_benchmark.py b/marvin-cli/tests/utils/test_benchmark.py
index d011f72..7d8d1a2 100755
--- a/marvin-cli/tests/utils/test_benchmark.py
+++ b/marvin-cli/tests/utils/test_benchmark.py
@@ -68,7 +68,7 @@ mocked_data = {
def test_create_or_get_benchmark_folder():
path = create_or_get_benchmark_folder()
- assert path == os.path.join(os.environ['MARVIN_HOME'], 'benchmarks')
+ assert path == os.path.join(os.environ['MARVIN_DATA_PATH'], 'benchmarks')
def test_create_or_return_poi():
timestamp = 'mocked_timestamp'
[incubator-marvin] 02/04: Fix .travis.yml
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit 212a84b3770cfe3927e8808d8dac7f0de5c0be92
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Thu Oct 21 20:20:58 2021 -0300
Fix .travis.yml
---
.travis.yml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 958de0b..9ead5a1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -58,6 +58,7 @@ matrix:
- travis_retry pip install unidecode
- travis_retry pip install tox
- make install
+ - travis_retry pip install 'h5py==2.10.0' --force-reinstall
script:
- tox
- codecov
@@ -78,8 +79,8 @@ matrix:
- export MARVIN_DATA_PATH=./marvin_data
- export SPARK_HOME=../spark-2.1.1-bin-hadoop2.6
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libsasl2-dev python-pip graphviz -y ; fi
- - travis_retry sudo pip install --upgrade pip
- - travis_retry sudo pip install virtualenvwrapper --ignore-installed six
+ - travis_retry pip install --upgrade pip
+ - travis_retry pip install virtualenvwrapper --ignore-installed six
- source virtualenvwrapper.sh
install:
- travis_retry pip install codecov
[incubator-marvin] 01/04: Quick fixes at CLI and Airflow/Kubernetes
templates
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit f5f96396fd6cb7ec64ef3e96ae85fb80153eec28
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Oct 20 20:36:23 2021 -0300
Quick fixes at CLI and Airflow/Kubernetes templates
---
marvin-cli/MANIFEST.in | 2 +
marvin-cli/Makefile | 0
marvin-cli/README.md | 0
marvin-cli/bin/marvin | 59 ++--
marvin-cli/bin/marvin-api | 50 +++
marvin-cli/bin/marvin_complete | 0
marvin-cli/marvin_cli/VERSION | 0
marvin-cli/marvin_cli/__init__.py | 0
marvin-cli/marvin_cli/communication/__init__.py | 0
.../marvin_cli/communication/remote_calls.py | 18 +-
.../marvin_cli/communication/stubs/__init__.py | 0
.../marvin_cli/communication/stubs/daemon_pb2.py | 0
.../communication/stubs/daemon_pb2_grpc.py | 0
marvin-cli/marvin_cli/management/__init__.py | 0
.../management/airflow_template/cookiecutter.json | 8 +
.../{{cookiecutter.dag_name}}.py | 53 +++
marvin-cli/marvin_cli/management/edit.py | 6 -
marvin-cli/marvin_cli/management/engine.py | 379 ++++++++++++++++-----
marvin-cli/marvin_cli/management/generate.py | 56 +++
.../kubernetes_template/cookiecutter.json | 9 +
.../{{cookiecutter.engine_name}}/deploy.sh | 6 +
.../{{cookiecutter.engine_name}}_deployment.yaml | 21 ++
.../{{cookiecutter.engine_name}}_service.yaml | 11 +
marvin-cli/marvin_cli/management/notebook.py | 34 +-
.../management/template/cookiecutter.json | 0
.../{{cookiecutter.project_name}}/.bumpversion.cfg | 0
.../{{cookiecutter.project_name}}/.coveragerc | 0
.../{{cookiecutter.project_name}}/.gitignore | 0
.../{{cookiecutter.project_name}}/CHANGES.md | 0
.../template/{{cookiecutter.project_name}}/INSTALL | 0
.../template/{{cookiecutter.project_name}}/LICENSE | 0
.../{{cookiecutter.project_name}}/MANIFEST.in | 3 +-
.../{{cookiecutter.project_name}}/Makefile | 0
.../{{cookiecutter.project_name}}/README.md | 0
.../docker/{develop => deploy}/daemon/Dockerfile | 76 +++--
.../docker/deploy/daemon/deps/start.sh | 2 +
.../docker/develop/daemon/Dockerfile | 76 +++--
.../{{cookiecutter.project_name}}/docs.yaml | 0
.../{{cookiecutter.project_name}}/engine.messages | 0
.../{{cookiecutter.project_name}}/engine.metadata | 0
.../{{cookiecutter.project_name}}/engine.params | 0
.../feedback.messages | 0
.../{{cookiecutter.project_name}}/marvin.ini | 0
.../notebooks/sample.ipynb | 0
.../{{cookiecutter.project_name}}/pytest.ini | 0
.../scripts/install_automl.sh | 0
.../{{cookiecutter.project_name}}/setup.cfg | 0
.../{{cookiecutter.project_name}}/setup.py | 0
.../tests/conftest.py | 0
.../data_handler/test_acquisitor_and_cleaner.py | 0
.../tests/data_handler/test_training_preparator.py | 0
.../tests/prediction/test_feedback.py | 0
.../tests/prediction/test_prediction_preparator.py | 0
.../tests/prediction/test_predictor.py | 0
.../tests/training/test_metrics_evaluator.py | 0
.../tests/training/test_trainer.py | 0
.../template/{{cookiecutter.project_name}}/tox.ini | 0
.../{{cookiecutter.project_package}}/VERSION | 0
.../{{cookiecutter.project_package}}/__init__.py | 0
.../_compatibility.py | 0
.../{{cookiecutter.project_package}}/_logging.py | 0
.../data_handler/__init__.py | 0
.../data_handler/acquisitor_and_cleaner.py | 0
.../data_handler/training_preparator.py | 0
.../prediction/__init__.py | 0
.../prediction/feedback.py | 0
.../prediction/prediction_preparator.py | 0
.../prediction/predictor.py | 0
.../training/__init__.py | 0
.../training/metrics_evaluator.py | 0
.../training/trainer.py | 0
marvin-cli/marvin_cli/management/test.py | 30 +-
marvin-cli/marvin_cli/utils/__init__.py | 0
marvin-cli/marvin_cli/utils/api.py | 72 ++++
marvin-cli/marvin_cli/utils/benchmark.py | 14 +-
marvin-cli/marvin_cli/utils/config.py | 0
marvin-cli/marvin_cli/utils/docker.py | 260 ++++++++++++--
marvin-cli/marvin_cli/utils/git.py | 4 +-
marvin-cli/marvin_cli/utils/log.py | 29 +-
marvin-cli/marvin_cli/utils/misc.py | 173 +++++++---
marvin-cli/pytest.ini | 0
marvin-cli/setup.py | 9 +-
.../tests/communication/test_remote_calls.py | 0
marvin-cli/tests/management/test_edit.py | 0
marvin-cli/tests/management/test_engine.py | 0
marvin-cli/tests/utils/test_benchmark.py | 0
marvin-cli/tests/utils/test_config.py | 0
marvin-cli/tests/utils/test_docker.py | 0
marvin-cli/tests/utils/test_git.py | 0
marvin-cli/tests/utils/test_misc.py | 0
marvin-cli/tox.ini | 0
91 files changed, 1161 insertions(+), 299 deletions(-)
diff --git a/marvin-cli/MANIFEST.in b/marvin-cli/MANIFEST.in
old mode 100644
new mode 100755
index 38394a0..1c905b6
--- a/marvin-cli/MANIFEST.in
+++ b/marvin-cli/MANIFEST.in
@@ -1,3 +1,5 @@
include README.md
include marvin_cli/VERSION
graft marvin_cli/management/template
+graft marvin_cli/management/airflow_template
+graft marvin_cli/management/kubernetes_template
diff --git a/marvin-cli/Makefile b/marvin-cli/Makefile
old mode 100644
new mode 100755
diff --git a/marvin-cli/README.md b/marvin-cli/README.md
old mode 100644
new mode 100755
diff --git a/marvin-cli/bin/marvin b/marvin-cli/bin/marvin
old mode 100644
new mode 100755
index 0a1b554..6bafc47
--- a/marvin-cli/bin/marvin
+++ b/marvin-cli/bin/marvin
@@ -17,8 +17,8 @@
import os
import click
-from marvin_cli.utils.misc import package_to_name
-from marvin_cli.utils.misc import kill_persisted_process
+from marvin_cli.utils.misc import package_to_name, create_tmp_marvin_folder
+from marvin_cli.utils.misc import retrieve_tmp_info
from marvin_cli.utils.config import parse_ini
from marvin_cli.utils.config import read_cli_conf, generate_default_conf
from marvin_cli.utils.docker import search_engine_container
@@ -29,18 +29,19 @@ from marvin_cli.management.engine import cli as cli_engine
from marvin_cli.management.notebook import cli as cli_notebook
from marvin_cli.management.edit import cli as cli_edit
from marvin_cli.management.test import cli as cli_test
+from marvin_cli.management.generate import cli as cli_generate
EXCLUDE_BY_TYPE = {
'toolbox': ['engine-dryrun', 'engine-grpcserver', 'engine-httpserver', 'notebook',
'lab', 'project-export', 'engine-logs', 'edit-metadata', 'test', 'test-tdd',
- 'test-tox', 'engine-bumpversion', 'benchmark', 'benchmark-plot'],
- 'engine': ['project-import']
+ 'test-tox', 'engine-bumpversion', 'benchmark', 'benchmark-plot', 'engine',
+ 'data', 'push', 'stop', 'clone', 'kube-deployment'],
+ 'engine': ['project-import', 'setup']
}
ENVIRONMENT_VARIABLES = {
'MARVIN_HOME': "{0}/marvin".format(os.environ['HOME']),
- 'MARVIN_DATA_PATH': "{0}/marvin/data".format(os.environ['HOME']),
- 'MARVIN_LOG': "{0}/marvin/data/.logs".format(os.environ['HOME'])
+ 'MARVIN_DATA_PATH': "{0}/marvin/data".format(os.environ['HOME'])
}
def _define_environ():
@@ -49,26 +50,18 @@ def _define_environ():
os.environ[var] = ENVIRONMENT_VARIABLES[var]
def _create_folders():
- _exports_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
_conf_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf')
_artifacts_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], '.artifacts')
+ _exports_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
_paths_list = [os.environ['MARVIN_HOME'], os.environ['MARVIN_DATA_PATH'],
- os.environ['MARVIN_LOG'], _exports_folder, _conf_folder, _artifacts_folder]
+ _exports_folder, _conf_folder, _artifacts_folder]
for path in _paths_list:
if not os.path.exists(path):
os.makedirs(path)
-def _initial_check_engine_mode(engine_package):
- if not search_engine_images(engine_package):
- if click.confirm('Do you want to create the engine docker image?', default=True):
- create_engine_image(engine_package)
- if not search_engine_container(engine_package):
- if click.confirm('Do you want to create the engine docker container?', default=True):
- create_daemon_container(engine_package, package_to_name(engine_package))
-
-def _read_config(engine_package):
+def _read_config():
filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
if not os.path.isfile(filepath):
conf = generate_default_conf()
@@ -83,60 +76,48 @@ sigh = """___ ___ _ _____ _ _____
| | | | (_| | | \ V /| | | | | | \__/\| |_____| |_
\_| |_/\__,_|_| \_/ |_|_| |_| \____/\_____/\___/\n\n"""
-ini_path = os.path.join(os.getcwd(), 'marvin.ini')
-config = None
-package_name = None
+engine_name = None
exclude = None
print(sigh)
_define_environ()
_create_folders()
+create_tmp_marvin_folder()
-if os.path.exists(ini_path):
- config = parse_ini(ini_path)
- package_name = config['marvin_package']
+engine = retrieve_tmp_info('engine')
+if engine is not None:
exclude = EXCLUDE_BY_TYPE['engine']
- print("[+] engine mode - {}".format(package_name))
- _initial_check_engine_mode(package_name)
+ engine_name = engine
+ print("[+] engine mode - {}".format(engine_name))
print("\n")
else:
exclude = EXCLUDE_BY_TYPE['toolbox']
print("[+] toolbox mode\n")
-config = _read_config(package_name)
+config = _read_config()
@click.group('main')
@click.pass_context
def cli(ctx):
ctx.obj = {
- 'package_name': package_name,
+ 'engine_name': engine_name,
'editor': config['editor'],
'default_host': config['default_host'],
'executor_url': config['executor_url']
}
-@cli.command("system-kill", help="Kill all marvin cli background process.")
-def kill():
- kill_persisted_process()
-
-
commands = {}
commands.update(cli_engine.commands)
commands.update(cli_notebook.commands)
commands.update(cli_edit.commands)
commands.update(cli_test.commands)
+commands.update(cli_generate.commands)
for name, command in commands.items():
if name not in exclude:
cli.add_command(command, name=name)
-cli()
-
-
-
-
-
-
+cli()
\ No newline at end of file
diff --git a/marvin-cli/bin/marvin-api b/marvin-cli/bin/marvin-api
new file mode 100644
index 0000000..2d42614
--- /dev/null
+++ b/marvin-cli/bin/marvin-api
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import marvin_cli.utils.api as api
+
+@click.group('main')
+def cli():
+ pass
+
+@cli.command("acquisitor")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def acquisitor(host, wait):
+ print("{0}/{1}".format(host, 'aq'))
+ api.run_acquisitor(host, wait)
+
+@cli.command("tpreparator")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def tpreparator(host, wait):
+ api.run_tpreparator(host, wait)
+
+@cli.command("trainer")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def trainer(host, wait):
+ api.run_trainer(host, wait)
+
+@cli.command("evaluator")
+@click.option('--host', '-h', default="https://127.0.0.1:8000")
+@click.option('--wait', '-w', default=False, is_flag=True)
+def evaluator(host, wait):
+ api.run_evaluator(host, wait)
+
+cli()
\ No newline at end of file
diff --git a/marvin-cli/bin/marvin_complete b/marvin-cli/bin/marvin_complete
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/VERSION b/marvin-cli/marvin_cli/VERSION
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/__init__.py b/marvin-cli/marvin_cli/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/__init__.py b/marvin-cli/marvin_cli/communication/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/remote_calls.py b/marvin-cli/marvin_cli/communication/remote_calls.py
old mode 100644
new mode 100755
index 9a11969..a9d7836
--- a/marvin-cli/marvin_cli/communication/remote_calls.py
+++ b/marvin-cli/marvin_cli/communication/remote_calls.py
@@ -27,13 +27,13 @@ class RemoteError(Exception):
pass
COMMANDS = {
- 'DRYRUN': daemon_pb2.Command.CommandType.DRYRUN,
- 'TEST': daemon_pb2.Command.CommandType.TEST,
- 'GRPC': daemon_pb2.Command.CommandType.GRPC,
- 'TDD': daemon_pb2.Command.CommandType.TDD,
- 'TOX': daemon_pb2.Command.CommandType.TOX,
- 'NOTEBOOK': daemon_pb2.Command.CommandType.NOTEBOOK,
- 'LAB': daemon_pb2.Command.CommandType.LAB
+ 'DRYRUN': daemon_pb2.Command.CommandType.Value('DRYRUN'),
+ 'TEST': daemon_pb2.Command.CommandType.Value('TEST'),
+ 'GRPC': daemon_pb2.Command.CommandType.Value('GRPC'),
+ 'TDD': daemon_pb2.Command.CommandType.Value('TDD'),
+ 'TOX': daemon_pb2.Command.CommandType.Value('TOX'),
+ 'NOTEBOOK': daemon_pb2.Command.CommandType.Value('NOTEBOOK'),
+ 'LAB': daemon_pb2.Command.CommandType.Value('LAB')
}
class RemoteCalls:
@@ -47,7 +47,7 @@ class RemoteCalls:
def call_command(self, name, parameters):
call = daemon_pb2.Command(command=COMMANDS[name], parameters=parameters)
response = self.stub.callCommand(call)
- if response.status == daemon_pb2.Status.NOK:
+ if response.status == daemon_pb2.Status.StatusType.Value('NOK'):
raise RemoteError("Error during {}.".format(name))
else:
logger.info("{} triggered!".format(name))
@@ -55,7 +55,7 @@ class RemoteCalls:
def stop_command(self, name):
call = daemon_pb2.Interruption()
response = self.stub.stopCommand(call)
- if response.status == daemon_pb2.Status.NOK:
+ if response.status == daemon_pb2.Status.StatusType.Value('NOK'):
raise RemoteError("Error during stop {}.".format(name))
else:
logger.info("{} stopped!".format(name))
diff --git a/marvin-cli/marvin_cli/communication/stubs/__init__.py b/marvin-cli/marvin_cli/communication/stubs/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/__init__.py b/marvin-cli/marvin_cli/management/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json b/marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json
new file mode 100644
index 0000000..d8c194e
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/airflow_template/cookiecutter.json
@@ -0,0 +1,8 @@
+{
+ "dag_name": "marvin-dag",
+ "host": "http://localhost:8000",
+ "owner": "marvin",
+ "interval_days": "1",
+ "timeout": "60",
+ "retries": "3"
+}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py b/marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py
new file mode 100644
index 0000000..1ada7fe
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/airflow_template/{{cookiecutter.dag_name}}/{{cookiecutter.dag_name}}.py
@@ -0,0 +1,53 @@
+import airflow
+from airflow import DAG
+from airflow.operators.bash_operator import BashOperator
+from datetime import timedelta
+
+HOST = "{{cookiecutter.host}}"
+
+args = {
+ 'owner': '{{cookiecutter.owner}}',
+}
+
+dag = DAG(
+ dag_id='{{cookiecutter.interval_days}}',
+ default_args=args,
+ schedule_interval=timedelta(days={{cookiecutter.interval_days}}),
+ dagrun_timeout=timedelta(minutes={{cookiecutter.timeout}})
+)
+
+acquisitor = BashOperator(
+ task_id='acquisitor',
+ bash_command='marvin-api acquisitor --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
+
+tpreparator = BashOperator(
+ task_id='tpreparator',
+ bash_command='marvin-api tpreparator --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
+
+trainer = BashOperator(
+ task_id='trainer',
+ bash_command='marvin-api trainer --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
+
+evaluator = BashOperator(
+ task_id='evaluator',
+ bash_command='marvin-api evaluator --wait --host $HOST',
+ retries={{cookiecutter.retries}},
+ env={
+ 'HOST': HOST
+ },
+ dag=dag)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/edit.py b/marvin-cli/marvin_cli/management/edit.py
old mode 100644
new mode 100755
index 37b24b6..156c748
--- a/marvin-cli/marvin_cli/management/edit.py
+++ b/marvin-cli/marvin_cli/management/edit.py
@@ -27,9 +27,3 @@ def cli():
def config(ctx):
filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
os.system(ctx.obj['editor'] + ' ' + filepath)
-
-@cli.command("edit-metadata", help="Edit engine.metadata.")
-@click.pass_context
-def metadata(ctx):
- filepath = "engine.metadata"
- os.system(ctx.obj['editor'] + ' ' + filepath)
diff --git a/marvin-cli/marvin_cli/management/engine.py b/marvin-cli/marvin_cli/management/engine.py
old mode 100644
new mode 100755
index 00ab26f..14b1925
--- a/marvin-cli/marvin_cli/management/engine.py
+++ b/marvin-cli/marvin_cli/management/engine.py
@@ -23,17 +23,26 @@ import wget
import click
import pickle
import pathlib
+import shutil
import subprocess
+import getpass
from cookiecutter.main import cookiecutter
from shutil import which
+from ..utils.docker import DaemonManagement
+from ..utils.docker import search_engine_container, search_docker_volume, shutdown_and_delete_container
+from ..utils.docker import create_engine_image, create_deploy_image_and_push, create_docker_volume, create_daemon_container
+from ..utils.docker import create_executor_container, delete_image_and_volume
+from ..utils.docker import rename_image, create_tfserving_container
from ..communication.remote_calls import RemoteCalls
from ..utils.misc import package_folder, extract_folder, get_version
from ..utils.misc import call_logs, package_to_name, get_executor_path_or_download
-from ..utils.misc import generate_timestamp, persist_process
+from ..utils.misc import generate_timestamp, write_tmp_info, generate_keys
+from ..utils.misc import init_port_forwarding
from ..utils.git import git_init, bump_version
from ..utils.log import get_logger
from ..utils.benchmark import benchmark_thread, create_poi, make_graph
+
logger = get_logger('engine')
@click.group("engine")
@@ -45,42 +54,155 @@ def _validate_project_name(name):
TEMPLATE_FOLDER = os.path.join(pathlib.Path(__file__).parent.absolute(), 'template')
+def check_engine(engine):
+ _engine_path = os.path.join(os.environ['MARVIN_HOME'], engine)
+ daemon = DaemonManagement(engine)
+
+ if os.path.exists(_engine_path):
+ if click.confirm('Do you want to update the cached engine?', default=True):
+ shutil.rmtree(_engine_path)
+ daemon.clone_engine()
+ else:
+ logger.info("Caching engine...")
+ daemon.clone_engine()
+ logger.info("Caching engine... Done!")
+
@cli.command("project-generate", help="Generate engine project")
@click.option('--name', '-n', prompt='Project name', help='Engine name')
-@click.option('--description', '-d', prompt='Project description', help='Engine description')
+@click.option('--description', '-d', prompt='Project description', help='Engine description', default='Marvin project')
@click.option('--url', '-u', prompt='Project URL', help='Engine URL Address', default='marvin.apache.org')
-@click.option('--maintainer', '-m', prompt='Maintainer', help='Engine Maintainer')
-@click.option('--email', '-e', prompt='Maintainer E-mail', help='Engine maintainer e-mail')
-@click.option('--dest', '-f', envvar='MARVIN_HOME', type=click.Path(exists=True), help='Root folder path for the creation')
+@click.option('--maintainer', '-m', prompt='Maintainer', help='Engine Maintainer', default='Marvin-AI')
+@click.option('--email', '-e', prompt='Maintainer E-mail', help='Engine maintainer e-mail', default='dev@marvin.apache.org')
+@click.option(
+ '--ptype',
+ '-pt',
+ default='python',
+ type=click.Choice(['python', 'tfx']),
+ help='Project type: Regular python project or TFX.')
@click.option('--template', '-t', help='Base template for engine.', default=TEMPLATE_FOLDER, type=click.Path(exists=True))
-def generate(name, description, url, maintainer, email, dest, template):
+def generate(name, description, url, maintainer, email, ptype, template):
+ #create engine files in /tmp/marvin
+ _dest = '/tmp/marvin'
_processed_name = _validate_project_name(name)
-
_extras_dir = {
'project_name': _processed_name,
'project_package': 'marvin_' + _processed_name.lower(),
'project_url': url,
'project_description': description,
'maintainer_name': maintainer,
- 'maintainer_email': email
+ 'maintainer_email': email,
+ 'project_type': ptype
}
- print(TEMPLATE_FOLDER)
- _init_dir = os.path.join(dest, _processed_name)
- cookiecutter(template, output_dir=dest, extra_context=_extras_dir, no_input=True)
+ _init_dir = os.path.join(_dest, _processed_name)
+ cookiecutter(template, output_dir=_dest, extra_context=_extras_dir, no_input=True)
git_init(_init_dir)
- logger.info("Engine {0} created".format(_processed_name))
+ #generate and put key in docker context
+ _pubkey_path = generate_keys(_processed_name)
+ _new_pubkey_path = os.path.join(_init_dir, 'docker', 'develop', 'daemon', 'id_rsa.pub')
+ shutil.move(_pubkey_path, _new_pubkey_path)
+
+ logger.info("Engine {0} created in /tmp/marvin.".format(_processed_name))
+ create_engine_image(name, _init_dir)
+ logger.info("Removing temporary files.")
+ shutil.rmtree(_init_dir, ignore_errors=True)
+ logger.info("Engine creation done!")
EXPORT_PATH = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
+@cli.command("clone", help="Clone files from daemon container.")
+@click.pass_context
+def engine_clone(ctx):
+ daemon = DaemonManagement(ctx.obj['engine_name'])
+ daemon.clone_engine()
+ daemon.clone_artifacts()
+ daemon.clone_logs()
+
+@cli.command("push", help="Rewrite engine files of daemon container.")
+@click.option('--compress/--not-compress', '-c/-nc', default=True, is_flag=True, help='Compress the stream.')
+@click.pass_context
+def engine_push(ctx, compress):
+ daemon = DaemonManagement(ctx.obj['engine_name'])
+ daemon.push_engine(compress)
+
+@cli.command("data", help="Actions related to data folder of daemon container.")
+@click.option(
+ '--action',
+ '-a',
+ default='list',
+ type=click.Choice(['push', 'delete', 'list']),
+ help='Data folder action type')
+@click.option('--compress/--not-compress', '-c/-nc', default=True, is_flag=True, help='Compress the stream.')
+@click.pass_context
+def data_push(ctx, action, compress):
+ daemon = DaemonManagement(ctx.obj['engine_name'])
+ if action == 'list':
+ daemon.list_data_files()
+ elif action == 'push':
+ _path = click.prompt('File to push', type=click.Path(exists=True))
+ daemon.push_data(_path, compress)
+ else:
+ _path = click.prompt('File to delete')
+ daemon.delete_data(_path)
+
+@cli.command("setup", help="Setup docker container and volumes to development.")
+@click.argument('engine', nargs=1)
+@click.pass_context
+def setup(ctx, engine):
+ _engine_volume = "marvin-{}-vol".format(engine)
+
+ logger.info("Setting up engine components...")
+ if not search_docker_volume("marvin-log"):
+ create_docker_volume("marvin-log")
+ if not search_docker_volume("marvin-data"):
+ create_docker_volume("marvin-data")
+ if not search_docker_volume(_engine_volume):
+ create_docker_volume(_engine_volume)
+ if not search_engine_container(engine):
+ create_daemon_container(engine)
+ write_tmp_info('engine', engine)
+ logger.info("Setting up engine components... Done!")
+ logger.info("Attaching logs...")
+ call_logs(engine)
+ logger.info("Attaching logs... Done!")
+ logger.info("Enabling port forwarding...")
+ init_port_forwarding(engine, ctx.obj['default_host'], [50057], background=True)
+ logger.info("Enabling port forwarding... Done!")
+
+@cli.command("stop", help="Stop docker container and workon.")
+@click.option('--delete', '-d', default=False, is_flag=True, help='Delete engine files permanently.')
+@click.pass_context
+def stop(ctx, delete):
+ _container_name = "marvin-cont-{}".format(ctx.obj['engine_name'])
+ _lock_path = '/tmp/marvin/engine'
+
+ logger.info("Stopping and deleting engine container...")
+ shutdown_and_delete_container(_container_name)
+ os.remove(_lock_path)
+ logger.info("Stopping and deleting engine container... Done!")
+
+ if delete:
+ check = click.prompt('If you are sure, type the engine name: ')
+ if check == ctx.obj['engine_name']:
+ _image_name = "marvin-{}".format(ctx.obj['engine_name'])
+ _volume_name = "marvin-{}-vol".format(ctx.obj['engine_name'])
+ delete_image_and_volume(_image_name, _volume_name)
+ logger.warning("Deleting private key...")
+ _key_path = os.path.join(os.environ['MARVIN_DATA_PATH'], '.keys',
+ ctx.obj['engine_name'])
+ shutil.rmtree(_key_path)
+ logger.warning("Deleting private key... Done!")
+
@cli.command("project-export", help="Export engine project to a archive file.")
@click.option('--dest', '-d', default=EXPORT_PATH, type=click.Path(exists=True), help='Output folder.')
@click.pass_context
def export(ctx, dest):
- filename = os.path.join(dest, ctx.obj['package_name']
- + '-' + get_version(ctx.obj['package_name']) + ".tar.gz")
+ check_engine(ctx.obj['engine_name'])
+ path = os.path.join(os.environ['MARVIN_HOME'], ctx.obj['engine_name'])
+ filename = os.path.join(dest, ctx.obj['engine_name']
+ + '-' + get_version(path, ctx.obj['engine_name']) + ".tar.gz")
- package_folder(os.getcwd(), filename)
+ package_folder(path, filename)
@cli.command("project-import", help="Import engine project from archive file.")
@click.option('--file', '-f', type=click.Path(exists=True), help='Compressed Engine file.')
@@ -89,7 +211,7 @@ def import_project(file, dest):
extract_folder(file, dest)
@cli.command("engine-dryrun", help="Run engines in a standalone way.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option(
'--action',
@@ -98,12 +220,28 @@ def import_project(file, dest):
type=click.Choice(['all', 'acquisitor', 'tpreparator', 'trainer', 'evaluator', 'ppreparator', 'predictor', 'feedback']),
help='Marvin engine action name')
@click.option('--profiling', '-p', default=False, is_flag=True, help='Deterministic profiling of user code.')
-def dryrun(grpchost, grpcport, action, profiling):
+@click.pass_context
+def dryrun(ctx, grpchost, grpcport, action, profiling):
+ if not grpchost:
+ grpchost = 'localhost'
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_dryrun(action, profiling)
+def grpc_port_forwarding(engine_name, grpchost):
+ ports = [
+ 50051,
+ 50052,
+ 50053,
+ 50054,
+ 50055,
+ 50056,
+ ]
+ init_port_forwarding(engine_name, grpchost,
+ ports_list=ports)
+
@cli.command("engine-grpcserver", help="Run gRPC of given actions.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option(
'--action',
@@ -113,31 +251,22 @@ def dryrun(grpchost, grpcport, action, profiling):
help='Marvin engine action name')
@click.option('--max-workers', '-w', help='Max Workers', default=None)
@click.option('--max-rpc-workers', '-rw', help='Max gRPC Workers', default=None)
-def grpc(grpchost, grpcport, action, max_workers, max_rpc_workers):
+@click.pass_context
+def grpc(ctx, grpchost, grpcport, action, max_workers, max_rpc_workers):
+ if not grpchost:
+ grpchost = 'localhost'
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_grpc(action, max_workers, max_rpc_workers)
- try:
- while(True):
- time.sleep(100)
- except KeyboardInterrupt:
- rc.stop_grpc()
- logger.info("gRPC server terminated!")
-
-@cli.command("engine-logs", help="Show daemon execution.")
-@click.option('--follow', '-f', is_flag=True)
-@click.option('--tail', '-t', default=True, is_flag=True)
-@click.option('--buffer', '-b', default=20)
-@click.pass_context
-def docker_logs(ctx, follow, tail, buffer):
- p_logs = call_logs(ctx.obj['package_name'], follow, buffer)
- if follow:
- persist_process(p_logs)
+ grpc_port_forwarding(ctx.obj['engine_name'], ctx.obj['default_host'])
+ rc.stop_grpc()
+ logger.info("gRPC server terminated!")
@cli.command("engine-httpserver", help="Run executor HTTP server.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
-@click.option('--host', '-h', prompt='API host', help='REST API Host', default='localhost')
-@click.option('--port', '-p', prompt='API port', help='REST API Port', default='8000')
+@click.option('--host', '-h', help='REST API Host', default='localhost')
+@click.option('--port', '-p', help='REST API Port', default='8000')
@click.option('--protocol', '-pr', help='Marvin protocol to be loaded during initialization.', default='')
@click.option(
'--action',
@@ -150,9 +279,16 @@ def docker_logs(ctx, follow, tail, buffer):
@click.option('--executor-path', '-e', help='Marvin engine executor jar path', type=click.Path(exists=True))
@click.option('--extra-executor-parameters', '-jvm', help='Use to send extra JVM parameters to engine executor process')
@click.option('--benchmark', '-b', default=False, is_flag=True, help='Run benchmark.')
+@click.option('--no-docker', '-nd', default=False, is_flag=True, help='Don\'t run the engine-executor on a Docker container.')
@click.pass_context
def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
- max_rpc_workers, executor_path, extra_executor_parameters, benchmark):
+ max_rpc_workers, executor_path, extra_executor_parameters, benchmark, no_docker):
+
+ if not grpchost:
+ grpchost = 'localhost'
+
+ if not host:
+ host = ctx.obj['default_host']
rc = RemoteCalls(grpchost, grpcport)
@@ -164,51 +300,77 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
sys.exit(1)
bench_thread = None
+ httpserver = None
try:
- if not executor_path:
- executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
-
- command_list = ['java']
- command_list.append('-DmarvinConfig.engineHome={}'.format(os.getcwd()))
- command_list.append('-DmarvinConfig.ipAddress={}'.format(host))
- command_list.append('-DmarvinConfig.port={}'.format(port))
- command_list.append('-DmarvinConfig.protocol={}'.format(protocol))
-
- if extra_executor_parameters:
- command_list.append(extra_executor_parameters)
-
- command_list.append('-jar')
- command_list.append(executor_path)
-
- if benchmark:
- logger.info("Init benchmark...")
- timestamp = generate_timestamp()
- bench_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
- bench_thread.start()
-
- httpserver = subprocess.Popen(command_list)
-
- except:
+ if not no_docker:
+ create_executor_container(ctx.obj['engine_name'])
+ else:
+ if not executor_path:
+ executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
+
+ check_engine(ctx.obj['engine_name'])
+ engine_path = os.path.join(os.environ['MARVIN_HOME'], ctx.obj['engine_name'])
+
+ command_list = ['java']
+ command_list.append('-DmarvinConfig.engineHome={}'.format(engine_path))
+ command_list.append('-DmarvinConfig.ipAddress={}'.format(host))
+ command_list.append('-DmarvinConfig.port={}'.format(port))
+ command_list.append('-DmarvinConfig.protocol={}'.format(protocol))
+
+ if extra_executor_parameters:
+ command_list.append(extra_executor_parameters)
+
+ command_list.append('-jar')
+ command_list.append(executor_path)
+
+ if benchmark:
+ logger.info("Init benchmark...")
+ timestamp = generate_timestamp()
+ bench_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
+ bench_thread.start()
+
+ httpserver = subprocess.Popen(command_list)
+
+ while True:
+ try:
+ time.sleep(100)
+ except KeyboardInterrupt:
+ break
+
+ except Exception as e:
logger.error("Could not start http server!")
+ if not no_docker:
+ shutdown_and_delete_container("marvin-executor-" +
+ ctx.obj['engine_name'])
+ print(e)
rc.stop_grpc()
if benchmark:
bench_thread.terminate()
sys.exit(1)
-
- try:
- while True:
- time.sleep(100)
-
- except KeyboardInterrupt:
+
+ if no_docker:
logger.info("Terminating http and grpc servers...")
rc.stop_grpc()
httpserver.terminate() if httpserver else None
logger.info("Http and grpc servers terminated!")
- if benchmark:
- bench_thread.terminate()
+ else:
+ try:
+ while True:
+ time.sleep(100)
+
+ except KeyboardInterrupt:
+ logger.info("Terminating http and grpc servers...")
+ rc.stop_grpc()
+ shutdown_and_delete_container("marvin-executor-" +
+ ctx.obj['engine_name'])
+ logger.info("Http and grpc servers terminated!")
+
+ if benchmark:
+ bench_thread.terminate()
logger.info("Benchmark terminated!")
- sys.exit(0)
+
+ sys.exit(0)
@cli.command('engine-bumpversion', help='Bump, commit and tag engine version.')
@@ -220,8 +382,11 @@ def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
default='patch',
type=click.Choice(['major', 'minor', 'patch']),
help='The part of the version to increase.')
-def bumpversion(verbose, dry_run, part):
- bump_version(part, verbose, dry_run)
+@click.pass_context
+def bumpversion(ctx, verbose, dry_run, part):
+ check_engine(ctx.obj['engine_name'])
+ path = os.path.join(os.environ['MARVIN_HOME'], ctx.obj['engine_name'])
+ bump_version(path, part, verbose, dry_run)
POI_LABELS = {
'acquisitor': 'ac',
@@ -235,16 +400,20 @@ def _sleep(sec):
time.sleep(5)
@cli.command("benchmark", help="Collect engine benchmark stats.")
-@click.option('--grpchost', '-gh', help='gRPC Host Address', default='localhost')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--profiling', '-p', default=False, is_flag=True, help='Deterministic profiling of user code.')
@click.option('--delay', '-d', default=False, is_flag=True, help='Delay the benchmark for 5 seconds.')
@click.pass_context
def btest(ctx, grpchost, grpcport, profiling, delay):
timestamp = generate_timestamp()
- b_thread = benchmark_thread(ctx.obj['package_name'], timestamp)
+ b_thread = benchmark_thread(ctx.obj['engine_name'], timestamp)
actions = ['acquisitor', 'tpreparator', 'trainer',
'evaluator']
+
+ if not grpchost:
+ grpchost = 'localhost'
+
rc = RemoteCalls(grpchost, grpcport)
initial_time = time.time()
b_thread.start()
@@ -303,3 +472,61 @@ def plot(protocol):
make_graph(op_name, label, protocol)
except KeyboardInterrupt:
sys.exit(0)
+
+@cli.command("kube-deployment", help="Deploy Kubernetes production pod.")
+@click.option('--namespace', '-p', default='default', help='Define Kubernetes namespace.')
+@click.option('--nreplicas', '-nr', default=1, help='Define number of pod replicas.')
+@click.option('--name-in-registry', '-nr', prompt='Image name in registry', help='Docker registry repository to pull the deploy image.', default='')
+@click.option('--target-port', '-tp', help='Target port on cluster.', default="9736")
+@click.pass_context
+def kube_deploy(ctx, namespace, nreplicas, name_in_registry, target_port):
+ engine_name = ctx.obj['engine_name']
+ port = None
+ if not click.confirm('Do you want to use the TFX serving method?', default=False):
+ artifact_path = os.path.join(os.environ['MARVIN_DATA_PATH'],
+ '.artifacts', engine_name)
+ engine_path = os.path.join(os.environ['MARVIN_HOME'], engine_name)
+ deps_path = os.path.join(engine_path, 'docker', 'deploy', 'daemon', 'deps')
+ executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
+
+ daemon = DaemonManagement(engine_name)
+ check_engine(engine_name)
+ daemon.clone_artifacts()
+
+ shutil.move(artifact_path, os.path.join(deps_path, 'artifacts'))
+ shutil.copy(executor_path, deps_path)
+ create_deploy_image_and_push(engine_name, engine_path, name_in_registry)
+ port = "8000"
+ else:
+ model_path = click.prompt('Model path', type=str)
+ old_image_name = "tfserving:{}".format(engine_name)
+ create_tfserving_container(engine_name, model_path)
+ rename_image(old_image_name, name_in_registry)
+ logger.info("Pushing deploy image to registry...")
+ os.system("docker push {}".format(name_in_registry))
+ logger.info("Pushing deploy image to registry... Done!")
+ port = "8500"
+
+ kubernetes_template = os.path.join(pathlib.Path(__file__).parent.absolute(),
+ 'kubernetes_template')
+
+ _extras_dir = {
+ 'engine_name': engine_name,
+ 'n_reps': nreplicas,
+ 'image_name': name_in_registry,
+ 'service_name': "{}-service".format(engine_name),
+ 'deployment_name': "{}-deployment".format(engine_name),
+ 'target_port': target_port,
+ 'container_port': port
+ }
+
+ cookiecutter(kubernetes_template, output_dir=os.getcwd(),
+ extra_context=_extras_dir, no_input=True)
+
+
+
+
+
+
+
+
diff --git a/marvin-cli/marvin_cli/management/generate.py b/marvin-cli/marvin_cli/management/generate.py
new file mode 100644
index 0000000..0d70764
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/generate.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import click
+import pathlib
+from cookiecutter.main import cookiecutter
+from ..utils.log import get_logger
+
+logger = get_logger('management.generate')
+
+@click.group("generate")
+def cli():
+ pass
+
+AIRFLOW_TEMPLATE = os.path.join(pathlib.Path(__file__).parent.absolute(), 'airflow_template')
+
+@cli.command("generate-dag", help="Generate Airflow DAG.")
+@click.option('--name', '-n', prompt='DAG name', default='marvin-dag', help='DAG name')
+@click.option('--host', '-h', prompt='Host', default='http://localhost:8000',
+ help='Hostname with port.')
+@click.option('--owner', '-o', prompt='Owner', default='Marvin',
+ help='Owner of the DAG.')
+@click.option('--interval-days', '-i', prompt='Interval days', default='1',
+ help='DAG executiom interval in days.')
+@click.option('--timeout', '-t', prompt='Timeout', default='60',
+ help='Timeout in minutes.')
+@click.option('--retries', '-r', prompt='Retries', default='3',
+ help='# of task each retries.')
+@click.option('--path', '-p', help='Output path to file.',
+ default=os.getcwd(), type=click.Path(exists=True))
+def generate_dag(name, host, owner, interval_days, timeout, retries, path):
+ _dest = path
+ _extras_dir = {
+ 'dag_name': name,
+ 'host': host,
+ 'owner': owner,
+ 'interval_days': interval_days,
+ 'timeout': timeout,
+ 'retries': retries
+ }
+ cookiecutter(AIRFLOW_TEMPLATE, output_dir=_dest, extra_context=_extras_dir, no_input=True)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json b/marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json
new file mode 100755
index 0000000..b6bb8f7
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/cookiecutter.json
@@ -0,0 +1,9 @@
+{
+ "engine_name": "hello",
+ "n_reps": "1",
+ "image_name": "marvin_deployment",
+ "service_name": "marvin_service",
+ "deployment_name": "marvin_deployment",
+ "target_port": "9736",
+ "container_port": "8000"
+}
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh
new file mode 100644
index 0000000..4f23d69
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/deploy.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+kubectl apply -f $SCRIPTPATH/{{cookiecutter.engine_name}}_deployment.yaml
+kubectl apply -f $SCRIPTPATH/{{cookiecutter.engine_name}}_service.yaml
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml
new file mode 100644
index 0000000..47e91fb
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_deployment.yaml
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{cookiecutter.deployment_name}}
+ labels:
+ app: {{cookiecutter.deployment_name}}
+spec:
+ replicas: {{cookiecutter.n_reps}}
+ selector:
+ matchLabels:
+ app: {{cookiecutter.deployment_name}}
+ template:
+ metadata:
+ labels:
+ app: {{cookiecutter.deployment_name}}
+ spec:
+ containers:
+ - name: {{cookiecutter.deployment_name}}
+ image: {{cookiecutter.image_name}}
+ ports:
+ - containerPort: {{cookiecutter.container_port}}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml
new file mode 100644
index 0000000..9f049bf
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/kubernetes_template/{{cookiecutter.engine_name}}/{{cookiecutter.engine_name}}_service.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{cookiecutter.service_name}}
+spec:
+ selector:
+ app: {{cookiecutter.deployment_name}}
+ ports:
+ - protocol: TCP
+ port: {{cookiecutter.container_port}}
+ targetPort: {{cookiecutter.target_port}}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/notebook.py b/marvin-cli/marvin_cli/management/notebook.py
old mode 100644
new mode 100755
index 01a8488..820743d
--- a/marvin-cli/marvin_cli/management/notebook.py
+++ b/marvin-cli/marvin_cli/management/notebook.py
@@ -15,25 +15,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import sys
import click
+import time
+from ..utils.log import get_logger
from ..communication.remote_calls import RemoteCalls
+from ..utils.misc import init_port_forwarding
+
+logger = get_logger('management.notebook')
@click.group("notebook")
def cli():
pass
@cli.command("notebook", help="Run custom engine Jupyter Notebook.")
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
-@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
-def notebook(grpchost, grpcport, notebook_port):
- rc = RemoteCalls(grpchost, grpcport)
- rc.run_notebook(notebook_port)
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
+@click.option('--notebook-port', '-np', help='Notebook port', default='8888')
+@click.option('--no-port-forwarding', '-npf', is_flag=True, default=False,
+ help='Connect ports between this system and remote host with SSH tunnel')
+@click.pass_context
+def notebook(ctx, grpchost, grpcport, notebook_port, no_port_forwarding):
+ if not grpchost:
+ grpchost = 'localhost'
-@cli.command("lab", help="Run custom engine Jupyter Lab.")
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
-@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
-def lab(grpchost, grpcport, notebook_port):
rc = RemoteCalls(grpchost, grpcport)
- rc.run_lab(notebook_port)
+ rc.run_notebook(notebook_port)
+ if not no_port_forwarding:
+ time.sleep(5)
+ logger.info("Enabling port forwarding {0}:{0}...".format(notebook_port))
+ init_port_forwarding(ctx.obj['engine_name'], ctx.obj['default_host'],
+ ports_list=[int(notebook_port)])
+ sys.exit(0)
diff --git a/marvin-cli/marvin_cli/management/template/cookiecutter.json b/marvin-cli/marvin_cli/management/template/cookiecutter.json
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
old mode 100644
new mode 100755
index c6c58b1..a747779
--- a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
@@ -6,4 +6,5 @@ include README.md
include {{cookiecutter.project_package}}/VERSION
recursive-include notebooks *
prune notebooks/build
-recursive-include tests *
\ No newline at end of file
+recursive-include tests *
+include docker/deploy/daemon/deps/start.sh
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/Dockerfile
similarity index 54%
copy from marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
copy to marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/Dockerfile
index 8418ff2..6e5deb3 100644
--- a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/Dockerfile
@@ -4,32 +4,44 @@
FROM marvin-daemon:python
-MAINTAINER {{cookiecutter.maintainer_name}}
-
ENV SLEEP_MILLIS 0
+#############################################################
+# CREATE USER #
+#############################################################
+
+RUN useradd -m marvin
+
##############################################################
# Define all environment variables to be used
##############################################################
-ENV ENGINE={{cookiecutter.project_name}}
-ENV MARVIN_HOME=/opt/marvin
-ENV MARVIN_LOG=$MARVIN_HOME/log
-ENV MARVIN_DATA_PATH=/opt/marvin/data
+ENV MARVIN_HOME=/home/marvin/
+ENV MARVIN_LOG=/home/marvin/log
+ENV MARVIN_DATA_PATH=/home/marvin/data
ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
ENV SPARK_HOME=/opt/spark
+ENV MARVIN_ENGINE_NAME={{cookiecutter.project_name}}
ENV SPARK_CONF_DIR=$SPARK_HOME/conf
ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
ENV YARN_CONF_DIR=$SPARK_CONF_DIR
##############################################################
+# Put here the artifact protocol #
+ENV ARTIFACT=
+##############################################################
+
+
+ENV PYTHONUNBUFFERED=0
+
+##############################################################
# Create all folders needed
##############################################################
RUN mkdir -p $MARVIN_HOME && \
mkdir -p $MARVIN_DATA_PATH && \
- mkdir -p $MARVIN_ENGINE_HOME && \
- mkdir -p $MARVIN_LOG
+ mkdir -p $MARVIN_LOG && \
+ mkdir -p $MARVIN_DATA_PATH/.artifacts/$MARVIN_ENGINE_NAME
##############################################################
# Install Apache Spark
@@ -47,24 +59,34 @@ RUN mkdir -p $MARVIN_HOME && \
RUN mkdir -p $SPARK_CONF_DIR
##############################################################
-# INSTALL ENGINE FROM SOURCE #
-##############################################################
-
-COPY {{cookiecutter.project_package}}-0.0.1.tar.gz .
-RUN pip install {{cookiecutter.project_package}}-0.0.1.tar.gz
-
-##############################################################
-# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+# INSTALL ENGINE FROM SOURCE #
##############################################################
-RUN pip install --no-cache pandas \
-&& pip install --no-cache matplotlib \
-&& pip install --no-cache cython \
-&& pip install --no-cache numpy \
-&& pip install --no-cache scikit-learn \
-&& pip install --no-cache pytest \
-&& pip install --no-cache pytest-cov \
-&& pip install --no-cache pytest-watch
+ADD {{cookiecutter.project_package}}-0.0.1.tar.gz $MARVIN_HOME
+ADD deps/artifacts/$MARVIN_ENGINE_NAME/* $MARVIN_DATA_PATH/.artifacts/$MARVIN_ENGINE_NAME/
+ADD deps/marvin-engine-executor-assembly-*.jar $MARVIN_DATA_PATH/executor.jar
+ADD deps/start.sh $MARVIN_DATA_PATH/start.sh
+
+RUN cd $MARVIN_HOME && \
+mv {{cookiecutter.project_name}} engine && \
+cd $MARVIN_ENGINE_HOME && \
+pip3 install -e . --no-compile && \
+apt update && \
+apt install -y openjdk-8-jre && \
+update-alternatives --config java
+
+###############################################################
+# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+###############################################################
+
+RUN pip3 install --no-cache pandas \
+&& pip3 install --no-cache matplotlib \
+&& pip3 install --no-cache cython \
+&& pip3 install --no-cache numpy \
+&& pip3 install --no-cache scikit-learn \
+&& pip3 install --no-cache pytest \
+&& pip3 install --no-cache pytest-cov \
+&& pip3 install --no-cache pytest-watch
##############################################################
# Uninstalling unnecessary software and cleaning cache
@@ -76,7 +98,7 @@ RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
# Starts the daemon
##############################################################
-EXPOSE 50051-50057
-EXPOSE 8888
+EXPOSE 50051-50056
+EXPOSE 8000
-CMD /bin/bash -c "cd $MARVIN_ENGINE_HOME && marvin-daemon"
\ No newline at end of file
+ENTRYPOINT ["/bin/bash", "-c", "cd $MARVIN_DATA_PATH && bash start.sh"]
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh
new file mode 100644
index 0000000..bfa415e
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/deploy/daemon/deps/start.sh
@@ -0,0 +1,2 @@
+cd $MARVIN_ENGINE_HOME && nohup marvin-daemon &
+java -DmarvinConfig.engineHome=/home/marvin/engine -DmarvinConfig.ipAddress=0.0.0.0 -DmarvinConfig.port=8000 -DmarvinConfig.protocol=$PROTOCOL -jar $MARVIN_DATA_PATH/executor.jar
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
old mode 100644
new mode 100755
index 8418ff2..da3aa07
--- a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
@@ -4,23 +4,30 @@
FROM marvin-daemon:python
-MAINTAINER {{cookiecutter.maintainer_name}}
+LABEL MAINTAINER {{cookiecutter.maintainer_name}}
ENV SLEEP_MILLIS 0
+#############################################################
+# CREATE USER #
+#############################################################
+
+RUN useradd -m marvin
+
##############################################################
# Define all environment variables to be used
##############################################################
-ENV ENGINE={{cookiecutter.project_name}}
-ENV MARVIN_HOME=/opt/marvin
-ENV MARVIN_LOG=$MARVIN_HOME/log
-ENV MARVIN_DATA_PATH=/opt/marvin/data
+ENV MARVIN_HOME=/home/marvin/
+ENV MARVIN_LOG=/home/marvin/log
+ENV MARVIN_DATA_PATH=/home/marvin/data
ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
ENV SPARK_HOME=/opt/spark
+ENV MARVIN_ENGINE_NAME={{cookiecutter.project_name}}
ENV SPARK_CONF_DIR=$SPARK_HOME/conf
ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+ENV PYTHONUNBUFFERED=0
##############################################################
# Create all folders needed
@@ -28,8 +35,8 @@ ENV YARN_CONF_DIR=$SPARK_CONF_DIR
RUN mkdir -p $MARVIN_HOME && \
mkdir -p $MARVIN_DATA_PATH && \
- mkdir -p $MARVIN_ENGINE_HOME && \
- mkdir -p $MARVIN_LOG
+ mkdir -p $MARVIN_LOG && \
+ mkdir -p $MARVIN_HOME/.ssh
##############################################################
# Install Apache Spark
@@ -46,25 +53,49 @@ RUN mkdir -p $MARVIN_HOME && \
RUN mkdir -p $SPARK_CONF_DIR
+###############################################################
+# CONFIGURE SSH SERVER #
+###############################################################
+
+ADD id_rsa.pub $MARVIN_HOME/.ssh/
+RUN apt update && \
+apt install -y net-tools openssh-server sudo && \
+sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/g' /etc/ssh/sshd_config && \
+sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+sed -i 's/PrintModt yes/PrintModt no/' /etc/ssh/sshd_config && \
+cat $MARVIN_HOME/.ssh/id_rsa.pub >> $MARVIN_HOME/.ssh/authorized_keys && \
+service ssh start && \
+chown -R marvin:marvin /home/marvin && \
+chmod -R 700 $MARVIN_HOME/.ssh && \
+chmod -R 600 $MARVIN_HOME/.ssh/authorized_keys && \
+echo 'marvin:Marvin' | chpasswd && \
+addgroup marvin staff && \
+addgroup marvin sudo && \
+true
+
##############################################################
-# INSTALL ENGINE FROM SOURCE #
+# INSTALL ENGINE FROM SOURCE #
##############################################################
-COPY {{cookiecutter.project_package}}-0.0.1.tar.gz .
-RUN pip install {{cookiecutter.project_package}}-0.0.1.tar.gz
+ADD {{cookiecutter.project_package}}-0.0.1.tar.gz $MARVIN_HOME
-##############################################################
-# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
-##############################################################
+RUN cd $MARVIN_HOME && \
+mv {{cookiecutter.project_name}} engine && \
+cd $MARVIN_ENGINE_HOME && \
+pip3 install -e . --no-compile
+
+###############################################################
+# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+###############################################################
-RUN pip install --no-cache pandas \
-&& pip install --no-cache matplotlib \
-&& pip install --no-cache cython \
-&& pip install --no-cache numpy \
-&& pip install --no-cache scikit-learn \
-&& pip install --no-cache pytest \
-&& pip install --no-cache pytest-cov \
-&& pip install --no-cache pytest-watch
+RUN pip3 install --no-cache pandas \
+&& pip3 install --no-cache matplotlib \
+&& pip3 install --no-cache cython \
+&& pip3 install --no-cache numpy \
+&& pip3 install --no-cache scikit-learn \
+&& pip3 install --no-cache pytest \
+&& pip3 install --no-cache pytest-cov \
+&& pip3 install --no-cache pytest-watch
##############################################################
# Uninstalling unnecessary software and cleaning cache
@@ -77,6 +108,7 @@ RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
##############################################################
EXPOSE 50051-50057
+EXPOSE 22
EXPOSE 8888
-CMD /bin/bash -c "cd $MARVIN_ENGINE_HOME && marvin-daemon"
\ No newline at end of file
+CMD /bin/bash -c "/usr/sbin/sshd && cd $MARVIN_ENGINE_HOME && nohup marvin-daemon"
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/management/test.py b/marvin-cli/marvin_cli/management/test.py
old mode 100644
new mode 100755
index 8d3cded..c0a7224
--- a/marvin-cli/marvin_cli/management/test.py
+++ b/marvin-cli/marvin_cli/management/test.py
@@ -23,32 +23,44 @@ def cli():
pass
@cli.command('test', help='Run tests.')
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--cov/--no-cov', default=True)
@click.option('--no-capture', is_flag=True)
@click.option('--pdb', is_flag=True)
@click.argument('args', default='')
-def test(grpchost, grpcport, cov, no_capture, pdb, args):
+@click.pass_context
+def test(ctx, grpchost, grpcport, cov, no_capture, pdb, args):
+ if not grpchost:
+ grpchost = ctx.obj['default_host']
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_test(cov, no_capture, pdb, args)
@cli.command('test-tox', help='Run tests using Tox environment.')
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.argument('args', default='--current-env')
-def tox(grpchost, grpcport, args):
+@click.pass_context
+def tox(ctx, grpchost, grpcport, args):
+ if not grpchost:
+ grpchost = ctx.obj['default_host']
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_tox(args)
@cli.command('test-tdd', help='Watch for changes to run tests automatically.')
-@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
-@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--grpchost', '-gh', help='gRPC Host Address', default=None)
+@click.option('--grpcport', '-gp', help='gRPC Port', default='50057')
@click.option('--cov/--no-cov', default=False)
@click.option('--no-capture', is_flag=True)
@click.option('--pdb', is_flag=True)
@click.option('--partial', is_flag=True)
@click.argument('args', default='')
-def tdd(grpchost, grpcport, cov, no_capture, pdb, partial, args):
+@click.pass_context
+def tdd(ctx, grpchost, grpcport, cov, no_capture, pdb, partial, args):
+ if not grpchost:
+ grpchost = ctx.obj['default_host']
+
rc = RemoteCalls(grpchost, grpcport)
rc.run_tdd(cov, no_capture, pdb, partial, args)
diff --git a/marvin-cli/marvin_cli/utils/__init__.py b/marvin-cli/marvin_cli/utils/__init__.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/utils/api.py b/marvin-cli/marvin_cli/utils/api.py
new file mode 100644
index 0000000..d352bb0
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/api.py
@@ -0,0 +1,72 @@
+import json
+import time
+import requests
+import sys
+from .log import get_logger
+
+logger = get_logger('utils.api')
+
+class ApiError(Exception):
+ pass
+
+def run_action(host, action_name):
+ resp = requests.post("{0}/{1}".format(host, action_name), json={}, verify=False)
+ if resp.status_code != 200:
+ raise ApiError('POST /{0}/ {1}'.format(action_name, resp.status_code))
+ return resp.json()
+
+def wait_action(host, action_name, protocol):
+ retries = 3
+ while True:
+ payload = {
+ 'protocol': protocol
+ }
+ resp = requests.get("{0}/{1}/status".format(host, action_name),
+ verify=False, params=payload)
+ if resp.status_code != 200:
+ retries -= 1
+ if retries == 0:
+ raise ApiError('PUSH /{0}/status {1}'.format(action_name, resp.status_code))
+ continue
+ retries = 3
+ resp_dict = json.loads(resp.json()['result'])
+ if resp_dict['status']['name'] == 'finished':
+ break
+ else:
+ time.sleep(1)
+
+def run_acquisitor(host, wait=False):
+ try:
+ result = run_action(host, 'acquisitor')
+ if wait:
+ wait_action(host, 'acquisitor', result['result'])
+ except ApiError as e:
+ logger.error(e)
+ sys.exit(1)
+
+def run_tpreparator(host, wait=False):
+ try:
+ result = run_action(host, 'tpreparator')
+ if wait:
+ wait_action(host, 'tpreparator', result['result'])
+ except ApiError as e:
+ logger.error(e)
+ sys.exit(1)
+
+def run_trainer(host, wait=False):
+ try:
+ result = run_action(host, 'trainer')
+ if wait:
+ wait_action(host, 'trainer', result['result'])
+ except ApiError:
+ logger.error(e)
+ sys.exit(1)
+
+def run_evaluator(host, wait=False):
+ try:
+ result = run_action(host, 'evaluator')
+ if wait:
+ wait_action(host, 'evaluator', result['result'])
+ except ApiError:
+ logger.error(e)
+ sys.exit(1)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/benchmark.py b/marvin-cli/marvin_cli/utils/benchmark.py
old mode 100644
new mode 100755
index cff69d3..8c998c5
--- a/marvin-cli/marvin_cli/utils/benchmark.py
+++ b/marvin-cli/marvin_cli/utils/benchmark.py
@@ -31,7 +31,7 @@ logger = get_logger("benchmark")
#to-do
def create_or_get_benchmark_folder():
- _folder_path = os.path.join(os.getcwd(), 'benchmarks')
+ _folder_path = os.path.join(os.environ['MARVIN_DATA_PATH'], 'benchmarks')
if not os.path.exists(_folder_path):
os.makedirs(_folder_path)
return _folder_path
@@ -107,8 +107,8 @@ def filter_data(input_data, initial_time):
int(_r_net),
int(_t_net))
-def get_and_persist_stats(engine_package, initial_time, timestamp):
- _stats = get_stats(engine_package)
+def get_and_persist_stats(engine_name, initial_time, timestamp):
+ _stats = get_stats(engine_name)
_colleted_stats = filter_data(_stats, initial_time)
_filename = 'benchmark_{0}.csv'.format(timestamp)
_path = os.path.join(create_or_get_benchmark_folder(), _filename)
@@ -116,14 +116,14 @@ def get_and_persist_stats(engine_package, initial_time, timestamp):
writer = csv.writer(f)
writer.writerow(_colleted_stats)
-def repeat_stats_call(engine_package, timestamp, initial_time):
+def repeat_stats_call(engine_name, timestamp, initial_time):
while True:
getattr(sys.modules[__name__],
- 'get_and_persist_stats')(engine_package, initial_time, timestamp)
+ 'get_and_persist_stats')(engine_name, initial_time, timestamp)
-def benchmark_thread(engine_package, timestamp, initial_time=time.time()):
+def benchmark_thread(engine_name, timestamp, initial_time=time.time()):
return multiprocessing.Process(target=repeat_stats_call,
- args=(engine_package, timestamp, initial_time,))
+ args=(engine_name, timestamp, initial_time,))
def create_poi(key, value, timestamp):
_file_path = create_or_return_poi(timestamp)
diff --git a/marvin-cli/marvin_cli/utils/config.py b/marvin-cli/marvin_cli/utils/config.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/marvin_cli/utils/docker.py b/marvin-cli/marvin_cli/utils/docker.py
old mode 100644
new mode 100755
index 5a605b7..6dfaa62
--- a/marvin-cli/marvin_cli/utils/docker.py
+++ b/marvin-cli/marvin_cli/utils/docker.py
@@ -18,7 +18,9 @@
import os
from docker import DockerClient
from docker.errors import ImageNotFound, NotFound
-from .misc import package_to_name, generate_engine_package
+from shutil import copyfile
+from .misc import package_to_name, generate_engine_package, name_to_package
+from .misc import get_chunk_and_untar, get_tar_data
from ..utils.log import get_logger
logger = get_logger('docker')
@@ -28,6 +30,9 @@ def _get_client(env=True, url=None):
return DockerClient.from_env()
return DockerClient(base_url=url)
+def rename_image(old_name, new_name):
+ os.system("docker tag {} {}".format(old_name, new_name))
+
def _search_docker_image(name):
client = _get_client()
try:
@@ -44,48 +49,108 @@ def _search_docker_container(name):
except NotFound:
return False
-def search_engine_container(engine_package):
- container_name = "marvin-cont-" + package_to_name(engine_package)
+def search_docker_volume(name):
+ client = _get_client()
+ try:
+ client.volumes.get(name)
+ return True
+ except NotFound:
+ return False
+
+def search_engine_container(engine_name):
+ container_name = "marvin-cont-" + engine_name
return _search_docker_container(container_name)
-def search_engine_images(engine_package):
- image_name = "marvin-" + package_to_name(engine_package)
+def search_engine_images(engine_name):
+ image_name = "marvin-" + engine_name
return _search_docker_image(image_name)
-def create_engine_image(engine_package):
+def create_engine_image(engine_name, engine_path):
logger.info("Creating engine docker image ...")
- dockerfile_path = os.path.join(os.getcwd(), "docker",
+ _package = name_to_package(engine_name)
+ dockerfile_path = os.path.join(engine_path, "docker",
"develop", "daemon")
- generate_engine_package(engine_package)
+ generate_engine_package(_package, engine_path)
client = _get_client()
client.images.build(
path=dockerfile_path,
- tag="marvin-" + package_to_name(engine_package)
+ tag="marvin-" + engine_name
)
logger.info("Creating engine docker image ... Done!")
-def create_daemon_container(engine_package, engine_name):
+def create_deploy_image_and_push(engine_name, engine_path, name_in_registry):
+ logger.info("Creating engine deploy image ...")
+ _package = name_to_package(engine_name)
+ dockerfile_path = os.path.join(engine_path, "docker",
+ "deploy", "daemon")
+ generate_engine_package(_package, engine_path,
+ dest=dockerfile_path)
+
+ client = _get_client()
+ client.images.build(
+ path=dockerfile_path,
+ tag=name_in_registry
+ )
+ logger.info("Creating engine deploy image... Done!")
+ logger.info("Pushing deploy image to registry...")
+ os.system("docker push {}".format(name_in_registry))
+ logger.info("Pushing deploy image to registry... Done!")
+
+def create_docker_volume(volume_name):
+ client = _get_client()
+ logger.info("Creating {} docker volume...".format(volume_name))
+ client.volumes.create(volume_name)
+ logger.info("Creating {} docker volume... Done!".format(volume_name))
+
+def shutdown_and_delete_container(name):
+ client = _get_client()
+ try:
+ _container = client.containers.get(name)
+ _container.stop()
+ logger.warning("{} container was stopped!".format(name))
+ _container.remove()
+ logger.info("{} container was removed!".format(name))
+ except NotFound:
+ logger.warning("Container not found!")
+
+def delete_image_and_volume(name, vol):
+ client = _get_client()
+ try:
+ client.images.remove(name)
+ logger.warning("{} image was untagged, use prune to delete it permanently!".format(name))
+ _vol = client.volumes.get(vol)
+ _vol.remove()
+ logger.warning("{} volume was deleted!".format(vol))
+ except NotFound:
+ logger.warning("Image or volume not found!")
+
+def create_daemon_container(engine_name):
logger.info("Creating engine docker container ...")
+
client = _get_client()
- _engine_path = os.path.join(os.environ['MARVIN_HOME'], engine_name)
+ _engine_volume = "marvin-{}-vol".format(engine_name)
+ _data_volume = "marvin-data"
+ _log_volume = "marvin-log"
+
client.containers.run(
image="marvin-" + engine_name,
name="marvin-cont-" + engine_name,
volumes={
- _engine_path: {
- 'bind': '/opt/marvin/engine',
+ _engine_volume: {
+ 'bind': '/home/marvin/engine',
'mode': 'rw'
},
- os.environ['MARVIN_DATA_PATH']: {
- 'bind': '/opt/marvin/data',
+ _data_volume: {
+ 'bind': '/home/marvin/data',
'mode': 'rw'
},
- os.environ['MARVIN_LOG']: {
- 'bind': '/opt/marvin/log',
+ _log_volume: {
+ 'bind': '/home/marvin/log',
'mode': 'rw'
}
},
ports={
+ '22/tcp':2022,
'50051/tcp': 50051,
'50052/tcp': 50052,
'50053/tcp': 50053,
@@ -100,12 +165,165 @@ def create_daemon_container(engine_package, engine_name):
logger.info("Creating engine docker container ... Done!")
-def get_stats(engine_package):
+def create_executor_container(engine_name):
+ logger.info("Creating engine executor container ...")
+
+ client = _get_client()
+ _engine_volume = "marvin-{}-vol".format(engine_name)
+
+ client.containers.run(
+ image="marvin-executor",
+ name="marvin-executor-" + engine_name,
+ volumes={
+ _engine_volume: {
+ 'bind': '/home/marvin/engine',
+ 'mode': 'rw'
+ }
+ },
+ ports={
+ '8000/tcp': 8000
+ },
+ detach=True
+ )
+
+ logger.info("Creating executor docker container ... Done!")
+
+def create_tfserving_container(engine_name, model_path, serve=False):
+ logger.info("Creating temporary container ...")
+
+ _container_name = "tfserving-{}".format(engine_name)
+ _dest_path = "/models/{}".format(engine_name)
+
+ client = _get_client()
+ container = client.containers.run(
+ image="tensorflow/serving:1.11.1",
+ name=_container_name,
+ detach=True
+ )
+ logger.info("Creating temporary container ... Done!")
+
+ logger.info("Writing model on image ...")
+ os.system("docker cp {0} {1}:/models/{2}".format(model_path, _container_name, engine_name))
+
+ os.system(
+ "docker commit --change \'ENV MODEL_NAME {0}\' {1} tfserving:{0}"
+ .format(engine_name, _container_name)
+ )
+ logger.info("Writing model on image ... Done!")
+
+ logger.info("Cleaning...")
+ shutdown_and_delete_container(_container_name)
+ logger.info("Cleaning... Done!")
+
+ if serve:
+ logger.info("Serving...")
+ client.containers.run(
+ image="tensorflow/serving:{}".format(engine_name),
+ name=_container_name,
+ ports={
+ '8500/tcp': 8500
+ },
+ detach=True
+ )
+ logger.info("Serving... Done!")
+
+
+def get_stats(engine_name):
client = _get_client()
- if search_engine_container(engine_package):
- container_name = "marvin-cont-" + package_to_name(engine_package)
+ if search_engine_container(engine_name):
+ container_name = "marvin-cont-" + engine_name
container = client.containers.get(container_name)
return container.stats(stream=False)
else:
logger.error("Engine container was not found!")
- return None
\ No newline at end of file
+ return None
+
+class DaemonManagement:
+
+ data_path = '/home/marvin/data'
+ log_path = '/home/marvin/log'
+
+ def __init__(self, engine_name):
+ self.engine_name = engine_name
+ self.engine_path = '/home/marvin/engine'
+
+ @staticmethod
+ def get_container(engine_name):
+ _engine_container = "marvin-cont-{}".format(engine_name)
+ client = _get_client()
+ return client.containers.get(_engine_container)
+
+ def delete_from_daemon(self, source, folder):
+ if folder:
+ source = os.path.join(source, '*')
+
+ _container = self.get_container(self.engine_name)
+ _response = _container.exec_run("sh -c 'rm -rf {}'".format(source))
+
+ if _response.exit_code == 0:
+ logger.info("Files were deleted sucessfully!")
+ else:
+ logger.error("Error deleting files.")
+
+ def list_dir_from_daemon(self, source):
+ _container = self.get_container(self.engine_name)
+
+ _response = _container.exec_run("ls {}".format(source))
+ if _response.exit_code == 0:
+ print(_response.output.decode())
+ else:
+ logger.error("Error listing files.")
+
+ def copy_from_daemon(self, source, dest):
+ _container = self.get_container(self.engine_name)
+ bits, stats = _container.get_archive(source)
+ logger.info("Copying file from daemon container. Size:{} bytes...".format(stats['size']))
+ get_chunk_and_untar(bits, dest)
+ logger.info("Copying file from daemon container. Size:{} bytes... Done!".format(stats['size']))
+
+ def copy_to_daemon(self, source, dest, folder=True):
+ _container = self.get_container(self.engine_name)
+ data, filepath = get_tar_data(source, folder)
+
+ _response = _container.put_archive(dest, data)
+
+ if _response:
+ logger.info("File copy was sucessfull!")
+ else:
+ logger.error("Error in file copy.")
+ #delete tmp file
+ os.remove(filepath)
+
+ def push_data(self, path):
+ self.copy_to_daemon(path, self.data_path, compress, folder=False)
+
+ def delete_data(self, file):
+ _file_path = os.path.join(self.data_path, file)
+ self.delete_from_daemon(_file_path, folder=False)
+
+ def list_data_files(self):
+ self.list_dir_from_daemon(self.data_path)
+
+ def clone_engine(self):
+ _dest = os.environ['MARVIN_HOME']
+ _orig_folder_name = os.path.join(_dest, 'engine')
+ _mod_folder_name = os.path.join(_dest, self.engine_name)
+ self.copy_from_daemon(self.engine_path, _dest)
+ os.rename(_orig_folder_name, _mod_folder_name)
+
+ def push_engine(self):
+ _source = os.environ['MARVIN_HOME']
+ _mod_folder_name = os.path.join(_source, self.engine_name)
+ _engine_folder = os.path.join(_source, 'engine')
+ os.rename(_mod_folder_name, _engine_folder)
+ self.delete_from_daemon(self.engine_path, folder=True)
+ self.copy_to_daemon(_engine_folder, self.engine_path)
+
+ def clone_artifacts(self):
+ _dest = os.path.join(os.environ['MARVIN_DATA_PATH'], '.artifacts', self.engine_name)
+ _source = os.path.join(self.data_path, '.artifacts', self.engine_name)
+ self.copy_from_daemon(_source, _dest)
+
+ def clone_logs(self):
+ _dest = os.path.join(os.environ['MARVIN_DATA_PATH'], '.log', self.engine_name)
+ self.copy_from_daemon(self.log_path, _dest)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/git.py b/marvin-cli/marvin_cli/utils/git.py
old mode 100644
new mode 100755
index 6dec396..8eaff3b
--- a/marvin-cli/marvin_cli/utils/git.py
+++ b/marvin-cli/marvin_cli/utils/git.py
@@ -21,8 +21,8 @@ def git_init(path):
os.chdir(path)
os.system(' '.join(['git', 'init', '.']))
-def bump_version(part, verbose, dryrun):
- command = ['bump2version', part]
+def bump_version(path, part, verbose, dryrun):
+ command = ['bump2version', part, path]
if verbose:
command.append('--verbose')
if dryrun:
diff --git a/marvin-cli/marvin_cli/utils/log.py b/marvin-cli/marvin_cli/utils/log.py
old mode 100644
new mode 100755
index 2417977..a8a00a0
--- a/marvin-cli/marvin_cli/utils/log.py
+++ b/marvin-cli/marvin_cli/utils/log.py
@@ -26,12 +26,6 @@ import logging
DEFAULT_LOG_LEVEL = logging.INFO
-if os.environ.get('MARVIN_LOG') is None:
- DEFAULT_LOG_DIR = '/tmp'
-else:
- DEFAULT_LOG_DIR = os.environ['MARVIN_LOG']
-
-
class Logger(logging.getLoggerClass()):
"""Custom logger class.
@@ -51,13 +45,11 @@ logging.setLoggerClass(Logger)
def get_logger(name, namespace='marvin_ai',
- log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ log_level=DEFAULT_LOG_LEVEL):
"""Build a logger that outputs to a file and to the console,"""
log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
os.getenv('LOG_LEVEL', log_level))
- log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
- os.getenv('LOG_DIR', log_dir))
logger = logging.getLogger('{}.{}'.format(namespace, name))
logger.setLevel(log_level)
@@ -71,23 +63,4 @@ def get_logger(name, namespace='marvin_ai',
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
- try:
- log_path = os.path.abspath(log_dir)
- log_filename = '{name}.{pid}.log'.format(
- name=namespace, pid=os.getpid())
-
- file_path = str(os.path.join(log_path, log_filename))
-
- if not os.path.exists(log_path):
- os.makedirs(log_path)
-
- # Create a file handler
- file_handler = logging.FileHandler(file_path)
- file_handler.setLevel(log_level)
- file_handler.setFormatter(formatter)
- logger.addHandler(file_handler)
- except OSError as e:
- logger.error('Could not create log file {file}: {error}'.format(
- file=file_path, error=e.strerror))
-
return logger
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/misc.py b/marvin-cli/marvin_cli/utils/misc.py
old mode 100644
new mode 100755
index 84d7974..5f4d01b
--- a/marvin-cli/marvin_cli/utils/misc.py
+++ b/marvin-cli/marvin_cli/utils/misc.py
@@ -16,12 +16,18 @@
# limitations under the License.
import os
+import sys
import subprocess
import tarfile
import wget
import glob
import pickle
import datetime
+import time
+import shutil
+from cryptography.hazmat.primitives import serialization as crypto_serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.backends import default_backend as crypto_default_backend
from .log import get_logger
logger = get_logger('misc')
@@ -34,15 +40,23 @@ def package_to_name(package):
#remove marvin_ substring
return package[len("marvin_"):]
-def generate_engine_package(package):
- subprocess.run(['python', 'setup.py', 'sdist'])
- filename = package + "-" + get_version(package) + ".tar.gz"
- origin = os.path.join(os.getcwd(), "dist", filename)
- dest = os.path.join(os.getcwd(), "docker", "develop" ,"daemon", filename)
- os.rename(origin, dest)
+def name_to_package(name):
+ return "marvin_{}".format(name)
-def get_version(package):
- with open(os.path.join(os.getcwd(), package ,"VERSION"), 'rb') as f:
+def generate_engine_package(package, path, dest=None):
+ filename = package + "-" + get_version(package, path) + ".tar.gz"
+ output = os.path.join('/tmp/marvin', filename)
+ make_tarfile(output, path)
+
+ if dest is not None:
+ move_dest = os.path.join(dest, filename)
+ else:
+ move_dest = os.path.join(path, "docker", "develop" ,"daemon", filename)
+
+ shutil.move(output, move_dest)
+
+def get_version(package, path):
+ with open(os.path.join(path, package ,"VERSION"), 'rb') as f:
version = f.read().decode('ascii').strip()
return version
@@ -54,14 +68,9 @@ def extract_folder(input, output):
tf = tarfile.open(input)
tf.extractall(output)
-def call_logs(package, follow, buffer):
- container_name = 'marvin-cont-' + package_to_name(package)
- p_return = None
- if follow:
- p_return = subprocess.Popen(['docker', 'logs', '--follow', container_name], stdout=subprocess.PIPE)
- else:
- p_return = subprocess.Popen(['docker', 'logs', '--tail', str(buffer), container_name], stdout=subprocess.PIPE)
-
+def call_logs(engine):
+ container_name = 'marvin-cont-' + engine
+ p_return = subprocess.Popen(['docker', 'logs', '--follow', container_name], stdout=subprocess.PIPE)
return p_return
def create_or_return_tmp_dir():
@@ -70,27 +79,21 @@ def create_or_return_tmp_dir():
os.makedirs(tmp_path)
return tmp_path
-def kill_persisted_process():
- base_path = create_or_return_tmp_dir()
- for obj_file in glob.glob('{0}/*.mproc'.format(base_path)):
- pid = int(
- obj_file[(len(base_path) + 1):(len('.mproc') * -1)]
- )
- try:
- os.kill(pid, 9)
- logger.info("PID {0} now killed!".format(pid))
- except ProcessLookupError:
- logger.info("PID {0} already killed!".format(pid))
- os.remove(obj_file)
-
-
-def persist_process(obj):
- filepath = os.path.join(create_or_return_tmp_dir(),
- '{0}.mproc'.format(obj.pid))
- logger.info("Creating {0}...".format(filepath))
- with open(filepath, 'w'):
- pass
-
+def write_tmp_info(key, info):
+ _filepath = os.path.join(create_or_return_tmp_dir(), key)
+ logger.info("Creating {0}...".format(key))
+ with open(_filepath, 'w') as f:
+ f.write(info)
+
+def retrieve_tmp_info(key):
+ _filepath = os.path.join(create_or_return_tmp_dir(), key)
+ logger.info("Retriving {0}...".format(key))
+ try:
+ with open(_filepath, 'r') as f:
+ info = f.read()
+ return info
+ except:
+ return None
def get_executor_path_or_download(executor_url):
#get filename from url
@@ -105,4 +108,98 @@ def get_executor_path_or_download(executor_url):
return executor_path
def generate_timestamp():
- return datetime.datetime.now().timestamp()
\ No newline at end of file
+ return datetime.datetime.now().timestamp()
+
+def create_tmp_marvin_folder():
+ _dir = '/tmp/marvin'
+ if not os.path.exists(_dir):
+ os.makedirs(_dir)
+
+def get_chunk_and_untar(bits, output_path):
+ _dir = '/tmp/marvin'
+ _tmp_path = os.path.join(_dir, 'tmp_data')
+ #save tar in tmp file
+ with open(_tmp_path, 'wb') as f:
+ for chunk in bits:
+ f.write(chunk)
+ f.close()
+ #extract files
+ with tarfile.open(_tmp_path) as tf:
+ tf.extractall(output_path)
+ tf.close()
+ #remove tmp_data
+ os.remove(_tmp_path)
+
+def get_tar_data(source, folder, compress):
+ _dir = '/tmp/marvin'
+ tmp_path = os.path.join(_dir, 'tmp_data')
+ _tar_mode = "w:gz" if compress else "w"
+ #save tar in tmp file
+ with tarfile.open(tmp_path, _tar_mode) as tf:
+ if folder:
+ tf.add(source, arcname='.')
+ else:
+ tf.add(source, arcname=os.path.basename(source))
+
+ #get bytes from file
+ with open(tmp_path, 'rb') as bf:
+ temp_bytes = bf.read()
+ return (temp_bytes, tmp_path)
+
+def generate_keys(engine_name):
+ _key_path = os.path.join(os.environ['MARVIN_DATA_PATH'],
+ '.keys',
+ engine_name)
+
+ os.makedirs(_key_path)
+
+ pvk_path = os.path.join(_key_path, 'id_rsa')
+ pubk_path = os.path.join(_key_path, 'id_rsa.pub')
+
+ key = rsa.generate_private_key(
+ backend=crypto_default_backend(),
+ public_exponent=65537,
+ key_size=2048
+ )
+
+ private_key = key.private_bytes(
+ crypto_serialization.Encoding.PEM,
+ crypto_serialization.PrivateFormat.PKCS8,
+ crypto_serialization.NoEncryption()
+ )
+
+ public_key = key.public_key().public_bytes(
+ crypto_serialization.Encoding.OpenSSH,
+ crypto_serialization.PublicFormat.OpenSSH
+ )
+
+ open(pubk_path ,"w").write(public_key.decode("utf-8"))
+ open(pvk_path ,"w").write(private_key.decode("utf-8"))
+ os.chmod(pvk_path, 0o500)
+
+ return pubk_path
+
+def init_port_forwarding(engine_name, remote_host, ports_list, background=True):
+ if remote_host != 'localhost' and remote_host != '127.0.0.1':
+ pkey_path = os.path.join(os.environ['MARVIN_DATA_PATH'], '.keys', engine_name, 'id_rsa')
+ command_list = ["ssh"]
+ command_list.append("-o")
+ command_list.append("StrictHostKeyChecking=no")
+ command_list.append("-N")
+
+ if background:
+ command_list.append('-f')
+
+ for remote_port in ports_list:
+ command_list.append("-L")
+ command_list.append("localhost:{0}:localhost:{0}".format(remote_port))
+
+ command_list.append("-i")
+ command_list.append("{0}".format(pkey_path))
+ command_list.append("marvin@{0}".format(remote_host))
+ command_list.append("-p")
+ command_list.append("2022")
+ if not background:
+ logger.info("Press Ctrl+C to disable port forwarding")
+
+ os.system(" ".join(command_list))
\ No newline at end of file
diff --git a/marvin-cli/pytest.ini b/marvin-cli/pytest.ini
old mode 100644
new mode 100755
diff --git a/marvin-cli/setup.py b/marvin-cli/setup.py
old mode 100644
new mode 100755
index 82b3eba..5102069
--- a/marvin-cli/setup.py
+++ b/marvin-cli/setup.py
@@ -55,13 +55,18 @@ REQUIREMENTS_EXTERNAL = [
'click',
'cookiecutter',
'docker-compose',
- 'matplotlib'
+ 'matplotlib',
+ 'cryptography',
+ 'kubernetes',
+ 'requests'
]
# This is normally an empty list
DEPENDENCY_LINKS_EXTERNAL = []
# script to be used
-SCRIPTS = ['bin/marvin', 'bin/marvin_complete']
+SCRIPTS = ['bin/marvin',
+ 'bin/marvin_complete',
+ 'bin/marvin-api']
def _get_version():
diff --git a/marvin-cli/tests/communication/test_remote_calls.py b/marvin-cli/tests/communication/test_remote_calls.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/management/test_edit.py b/marvin-cli/tests/management/test_edit.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/management/test_engine.py b/marvin-cli/tests/management/test_engine.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_benchmark.py b/marvin-cli/tests/utils/test_benchmark.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_config.py b/marvin-cli/tests/utils/test_config.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_docker.py b/marvin-cli/tests/utils/test_docker.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_git.py b/marvin-cli/tests/utils/test_git.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tests/utils/test_misc.py b/marvin-cli/tests/utils/test_misc.py
old mode 100644
new mode 100755
diff --git a/marvin-cli/tox.ini b/marvin-cli/tox.ini
old mode 100644
new mode 100755