You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marvin.apache.org by we...@apache.org on 2020/08/29 07:38:28 UTC
[incubator-marvin] branch develop updated (ceb4f05 -> 716bd93)
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git.
from ceb4f05 Update tests to fix CI
new c32d30d Fix heathcheck bug
new e5dec0e [MARVIN-73] Add new cli and cookiecutter for templates
new 7f708a2 Change build folder to build daemon docker image
new 716bd93 Add cli in CI
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 29 ++
Makefile | 36 +-
build/daemon/Dockerfile | 8 +
build/marvin-engine-executor-assembly.jar | Bin 94184375 -> 0 bytes
build/python-toolbox.tgz | Bin 693248 -> 0 bytes
marvin-cli/MANIFEST.in | 3 +
{python-daemon => marvin-cli}/Makefile | 0
marvin-cli/README.md | 67 ++++
marvin-cli/bin/marvin | 137 ++++++++
.../marvin_cli}/VERSION | 0
.../stubs => marvin-cli/marvin_cli}/__init__.py | 0
.../marvin_cli/communication}/__init__.py | 0
.../marvin_cli/communication/remote_calls.py | 120 +++++++
.../marvin_cli/communication}/stubs/__init__.py | 0
.../marvin_cli/communication/stubs/daemon_pb2.py | 388 +++++++++++++++++++++
.../communication/stubs/daemon_pb2_grpc.py | 129 +++++++
.../marvin_cli/management}/__init__.py | 0
.../marvin_cli/management/edit.py | 19 +-
marvin-cli/marvin_cli/management/engine.py | 202 +++++++++++
marvin-cli/marvin_cli/management/notebook.py | 41 +++
.../management/template/cookiecutter.json | 10 +
.../{{cookiecutter.project_name}}/.bumpversion.cfg | 11 +
.../{{cookiecutter.project_name}}}/.coveragerc | 0
.../{{cookiecutter.project_name}}}/.gitignore | 0
.../{{cookiecutter.project_name}}}/CHANGES.md | 0
.../{{cookiecutter.project_name}}}/INSTALL | 0
.../{{cookiecutter.project_name}}}/LICENSE | 0
.../{{cookiecutter.project_name}}}/MANIFEST.in | 2 +-
.../{{cookiecutter.project_name}}}/Makefile | 34 +-
.../{{cookiecutter.project_name}}}/README.md | 10 +-
.../docker/develop/daemon/Dockerfile | 82 +++++
.../{{cookiecutter.project_name}}}/docs.yaml | 10 +-
.../{{cookiecutter.project_name}}}/engine.messages | 0
.../{{cookiecutter.project_name}}}/engine.metadata | 9 +-
.../{{cookiecutter.project_name}}}/engine.params | 0
.../feedback.messages | 0
.../{{cookiecutter.project_name}}/marvin.ini | 3 +
.../notebooks/sample.ipynb | 113 ++++++
.../{{cookiecutter.project_name}}}/pytest.ini | 0
.../scripts}/install_automl.sh | 0
.../{{cookiecutter.project_name}}}/setup.cfg | 0
.../{{cookiecutter.project_name}}}/setup.py | 66 +---
.../tests/conftest.py | 1 +
.../data_handler/test_acquisitor_and_cleaner.py | 3 +-
.../tests/data_handler/test_training_preparator.py | 4 +-
.../tests/prediction/test_feedback.py | 4 +-
.../tests/prediction/test_prediction_preparator.py | 4 +-
.../tests/prediction/test_predictor.py | 4 +-
.../tests/training/test_metrics_evaluator.py | 4 +-
.../tests/training/test_trainer.py | 3 +-
.../template/{{cookiecutter.project_name}}/tox.ini | 9 +
.../{{cookiecutter.project_package}}/VERSION | 1 +
.../{{cookiecutter.project_package}}}/__init__.py | 1 +
.../_compatibility.py | 1 +
.../{{cookiecutter.project_package}}}/_logging.py | 5 +-
.../data_handler/__init__.py | 2 +-
.../data_handler/acquisitor_and_cleaner.py | 13 +-
.../data_handler/training_preparator.py | 13 +-
.../prediction/__init__.py | 0
.../prediction/feedback.py | 2 +-
.../prediction/prediction_preparator.py | 6 +-
.../prediction/predictor.py | 11 +-
.../training/__init__.py | 0
.../training/metrics_evaluator.py | 13 +-
.../training/trainer.py | 17 +-
marvin-cli/marvin_cli/management/test.py | 54 +++
.../marvin_cli/utils}/__init__.py | 0
marvin-cli/marvin_cli/utils/config.py | 63 ++++
marvin-cli/marvin_cli/utils/docker.py | 101 ++++++
.../marvin_cli/utils/git.py | 15 +-
.../common => marvin-cli/marvin_cli/utils}/log.py | 8 +-
marvin-cli/marvin_cli/utils/misc.py | 64 ++++
{python-daemon => marvin-cli}/pytest.ini | 2 +-
{python-daemon => marvin-cli}/setup.py | 45 +--
.../tests/communication/test_remote_calls.py | 100 ++++++
.../tests/management/test_edit.py | 29 +-
.../tests/management/test_engine.py | 21 +-
marvin-cli/tests/utils/test_config.py | 46 +++
marvin-cli/tests/utils/test_docker.py | 67 ++++
.../tests/utils/test_git.py | 23 +-
marvin-cli/tests/utils/test_misc.py | 68 ++++
{python-daemon => marvin-cli}/tox.ini | 3 +-
.../engine_base/engine_base_action.py | 2 +-
.../engine_base/engine_base_data_handler.py | 8 +-
.../marvin_python_daemon/management/engine.py | 2 +-
python-daemon/setup.py | 6 +-
.../tests/engine_base/test_engine_base_action.py | 15 +-
.../engine_base/test_engine_base_data_handler.py | 2 +-
88 files changed, 2155 insertions(+), 239 deletions(-)
create mode 100644 build/daemon/Dockerfile
delete mode 100644 build/marvin-engine-executor-assembly.jar
delete mode 100644 build/python-toolbox.tgz
create mode 100644 marvin-cli/MANIFEST.in
copy {python-daemon => marvin-cli}/Makefile (100%)
create mode 100644 marvin-cli/README.md
create mode 100644 marvin-cli/bin/marvin
copy {python-daemon/marvin_python_daemon => marvin-cli/marvin_cli}/VERSION (100%)
copy {python-daemon/marvin_python_daemon/daemon_base/stubs => marvin-cli/marvin_cli}/__init__.py (100%)
copy {python-daemon/marvin_python_daemon/daemon_base/stubs => marvin-cli/marvin_cli/communication}/__init__.py (100%)
create mode 100644 marvin-cli/marvin_cli/communication/remote_calls.py
copy {python-daemon/marvin_python_daemon/daemon_base => marvin-cli/marvin_cli/communication}/stubs/__init__.py (100%)
create mode 100644 marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py
create mode 100644 marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py
copy {python-daemon/marvin_python_daemon/daemon_base/stubs => marvin-cli/marvin_cli/management}/__init__.py (100%)
copy python-daemon/marvin_python_daemon/__init__.py => marvin-cli/marvin_cli/management/edit.py (58%)
create mode 100644 marvin-cli/marvin_cli/management/engine.py
create mode 100644 marvin-cli/marvin_cli/management/notebook.py
create mode 100644 marvin-cli/marvin_cli/management/template/cookiecutter.json
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/.coveragerc (100%)
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/.gitignore (100%)
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/CHANGES.md (100%)
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/INSTALL (100%)
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/LICENSE (100%)
copy {public-engines/kaggle-titanic-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/MANIFEST.in (65%)
copy {public-engines/sample-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/Makefile (54%)
copy {public-engines/product-classifier-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/README.md (92%)
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/docs.yaml (98%)
copy {public-engines/sample-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/engine.messages (100%)
copy {public-engines/iris-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/engine.metadata (91%)
copy {public-engines/sample-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/engine.params (100%)
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/feedback.messages (100%)
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/pytest.ini (100%)
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts}/install_automl.sh (100%)
copy {public-engines/sms-spam-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/setup.cfg (100%)
copy {public-engines/iris-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/setup.py (62%)
copy {public-engines/iris-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/conftest.py (99%)
copy {public-engines/iris-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/data_handler/test_acquisitor_and_cleaner.py (79%)
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/data_handler/test_training_preparator.py (71%)
copy {public-engines/product-classifier-engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/prediction/test_feedback.py (74%)
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/prediction/test_prediction_preparator.py (73%)
copy {public-engines/iris-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/prediction/test_predictor.py (74%)
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/training/test_metrics_evaluator.py (72%)
copy {public-engines/iris-h2o-automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}}/tests/training/test_trainer.py (81%)
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini
create mode 100644 marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION
copy {public-engines/iris-automl/marvin_iris_automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/__init__.py (99%)
copy {public-engines/iris-species-engine/marvin_iris_species_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/_compatibility.py (99%)
copy {public-engines/iris-automl/marvin_iris_automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/_logging.py (96%)
copy {public-engines/image-classification-engine/marvin_image_classification_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/data_handler/__init__.py (64%)
copy {public-engines/iris-automl/marvin_iris_automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/data_handler/acquisitor_and_cleaner.py (61%)
copy {public-engines/iris-h2o-automl/marvin_iris_h2o_automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/data_handler/training_preparator.py (54%)
copy {public-engines/iris-species-engine/marvin_iris_species_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/prediction/__init__.py (100%)
copy {public-engines/sms-spam-engine/marvin_sms_spam_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/prediction/feedback.py (94%)
copy {public-engines/iris-species-engine/marvin_iris_species_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/prediction/prediction_preparator.py (67%)
copy {public-engines/iris-species-engine/marvin_iris_species_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/prediction/predictor.py (60%)
copy {public-engines/sms-spam-engine/marvin_sms_spam_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/training/__init__.py (100%)
copy {public-engines/iris-automl/marvin_iris_automl => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/training/metrics_evaluator.py (57%)
copy {public-engines/sms-spam-engine/marvin_sms_spam_engine => marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}}/training/trainer.py (54%)
create mode 100644 marvin-cli/marvin_cli/management/test.py
copy {python-daemon/marvin_python_daemon/daemon_base/stubs => marvin-cli/marvin_cli/utils}/__init__.py (100%)
create mode 100644 marvin-cli/marvin_cli/utils/config.py
create mode 100644 marvin-cli/marvin_cli/utils/docker.py
copy python-daemon/marvin_python_daemon/__init__.py => marvin-cli/marvin_cli/utils/git.py (68%)
copy {python-daemon/marvin_python_daemon/common => marvin-cli/marvin_cli/utils}/log.py (95%)
create mode 100644 marvin-cli/marvin_cli/utils/misc.py
copy {python-daemon => marvin-cli}/pytest.ini (63%)
copy {python-daemon => marvin-cli}/setup.py (77%)
create mode 100644 marvin-cli/tests/communication/test_remote_calls.py
copy public-engines/sample-engine/tests/data_handler/test_training_preparator.py => marvin-cli/tests/management/test_edit.py (53%)
copy public-engines/sample-engine/tests/data_handler/test_acquisitor_and_cleaner.py => marvin-cli/tests/management/test_engine.py (60%)
create mode 100644 marvin-cli/tests/utils/test_config.py
create mode 100644 marvin-cli/tests/utils/test_docker.py
copy public-engines/sample-engine/tests/data_handler/test_acquisitor_and_cleaner.py => marvin-cli/tests/utils/test_git.py (51%)
create mode 100644 marvin-cli/tests/utils/test_misc.py
copy {python-daemon => marvin-cli}/tox.ini (50%)
[incubator-marvin] 01/04: Fix heathcheck bug
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit c32d30dcb2323a1c5e3b156adf7fe538c5b1fe75
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Aug 26 17:35:26 2020 -0300
Fix heathcheck bug
---
.../engine_base/engine_base_action.py | 2 +-
.../engine_base/engine_base_data_handler.py | 8 ++++----
python-daemon/marvin_python_daemon/management/engine.py | 2 +-
python-daemon/setup.py | 6 +++---
.../tests/engine_base/test_engine_base_action.py | 15 +++++++++------
.../tests/engine_base/test_engine_base_data_handler.py | 2 +-
6 files changed, 19 insertions(+), 16 deletions(-)
diff --git a/python-daemon/marvin_python_daemon/engine_base/engine_base_action.py b/python-daemon/marvin_python_daemon/engine_base/engine_base_action.py
index 485b620..9db5b4f 100644
--- a/python-daemon/marvin_python_daemon/engine_base/engine_base_action.py
+++ b/python-daemon/marvin_python_daemon/engine_base/engine_base_action.py
@@ -155,7 +155,7 @@ class EngineBaseAction():
try:
if request.artifacts:
for artifact in request.artifacts.split(","):
- if getattr(self, artifact) is None:
+ if getattr(self, '_{0}'.format(artifact)) is None:
return HealthCheckResponse(status=HealthCheckResponse.NOK)
return HealthCheckResponse(status=HealthCheckResponse.OK)
diff --git a/python-daemon/marvin_python_daemon/engine_base/engine_base_data_handler.py b/python-daemon/marvin_python_daemon/engine_base/engine_base_data_handler.py
index 5e0dc8b..e0ace18 100644
--- a/python-daemon/marvin_python_daemon/engine_base/engine_base_data_handler.py
+++ b/python-daemon/marvin_python_daemon/engine_base/engine_base_data_handler.py
@@ -28,11 +28,11 @@ logger = get_logger('engine_base_data_handler')
class EngineBaseDataHandler(EngineBaseBatchAction):
__metaclass__ = ABCMeta
- _initial_dataset = None
+ _initialdataset = None
_dataset = None
def __init__(self, **kwargs):
- self._initial_dataset = self._get_arg(
+ self._initialdataset = self._get_arg(
kwargs=kwargs, arg='initial_dataset')
self._dataset = self._get_arg(kwargs=kwargs, arg='dataset')
super(EngineBaseDataHandler, self).__init__(**kwargs)
@@ -40,12 +40,12 @@ class EngineBaseDataHandler(EngineBaseBatchAction):
@property
def marvin_initial_dataset(self):
logger.info("initial_dataset loaded.")
- return self._load_obj(object_reference='_initial_dataset')
+ return self._load_obj(object_reference='_initialdataset')
@marvin_initial_dataset.setter
def marvin_initial_dataset(self, initial_dataset):
logger.info("initial_dataset saved.")
- self._save_obj(object_reference='_initial_dataset',
+ self._save_obj(object_reference='_initialdataset',
obj=initial_dataset)
@property
diff --git a/python-daemon/marvin_python_daemon/management/engine.py b/python-daemon/marvin_python_daemon/management/engine.py
index e12c452..425e802 100644
--- a/python-daemon/marvin_python_daemon/management/engine.py
+++ b/python-daemon/marvin_python_daemon/management/engine.py
@@ -45,7 +45,7 @@ ARTIFACTS = {
"AcquisitorAndCleaner": [],
"TrainingPreparator": ["initialdataset"],
"Trainer": ["dataset"],
- "MetricsEvaluator": ["model"],
+ "MetricsEvaluator": ["dataset", "model"],
"PredictionPreparator": ["model", "metrics"],
"Predictor": ["model", "metrics"],
"Feedback": []
diff --git a/python-daemon/setup.py b/python-daemon/setup.py
index 02f4f5d..b293767 100644
--- a/python-daemon/setup.py
+++ b/python-daemon/setup.py
@@ -27,10 +27,10 @@ from setuptools.command.test import test as TestCommand
PACKAGE_NAME = 'marvin_python_daemon'
PACKAGE_DESCRIPTION = 'Marvin Python Daemon'
-URL = ''
+URL = 'marvin.apache.org'
-AUTHOR_NAME = 'Lucas Cardoso'
-AUTHOR_EMAIL = ''
+AUTHOR_NAME = 'Apache Marvin-AI Community'
+AUTHOR_EMAIL = 'dev@marvin.apache.org'
PYTHON_2 = False
PYTHON_3 = True
diff --git a/python-daemon/tests/engine_base/test_engine_base_action.py b/python-daemon/tests/engine_base/test_engine_base_action.py
index 469c581..66251a9 100644
--- a/python-daemon/tests/engine_base/test_engine_base_action.py
+++ b/python-daemon/tests/engine_base/test_engine_base_action.py
@@ -139,21 +139,24 @@ class TestEngineBaseAction:
assert [1] == engine_action2._params
def test_health_check_ok(self, engine_action):
- obj1_key = "obj1"
- engine_action._save_obj(obj1_key, "check")
- request = HealthCheckRequest(artifacts=obj1_key)
+ obj = "obj1"
+ obj_key = '_{0}'.format(obj)
+ engine_action._save_obj(obj_key, "check")
+ request = HealthCheckRequest(artifacts=obj)
expected_response = HealthCheckResponse(status=HealthCheckResponse.OK)
response = engine_action._health_check(request=request, context=None)
assert expected_response.status == response.status
def test_health_check_ok_multiple(self, engine_action):
- obj1_key = "obj1"
+ obj1 = "obj1"
+ obj1_key = '_{0}'.format(obj1)
engine_action._save_obj(obj1_key, "check")
- obj2_key = "obj2"
+ obj2 = "obj2"
+ obj2_key = '_{0}'.format(obj2)
engine_action._save_obj(obj2_key, "check")
- request = HealthCheckRequest(artifacts=obj1_key + "," + obj2_key)
+ request = HealthCheckRequest(artifacts=obj1 + "," + obj2)
expected_response = HealthCheckResponse(status=HealthCheckResponse.OK)
response = engine_action._health_check(request=request, context=None)
diff --git a/python-daemon/tests/engine_base/test_engine_base_data_handler.py b/python-daemon/tests/engine_base/test_engine_base_data_handler.py
index 8e93ccc..0fb7a38 100644
--- a/python-daemon/tests/engine_base/test_engine_base_data_handler.py
+++ b/python-daemon/tests/engine_base/test_engine_base_data_handler.py
@@ -33,7 +33,7 @@ class TestEngineBaseDataHandler:
def test_initial_dataset(self, engine_action):
engine_action.marvin_initial_dataset = [1]
- assert engine_action.marvin_initial_dataset == engine_action._initial_dataset == [
+ assert engine_action.marvin_initial_dataset == engine_action._initialdataset == [
1]
def test_dataset(self, engine_action):
[incubator-marvin] 02/04: [MARVIN-73] Add new cli and cookiecutter
for templates
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit e5dec0ef31dd57a418372d404d842da4b52378fe
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Aug 26 17:37:22 2020 -0300
[MARVIN-73] Add new cli and cookiecutter for templates
---
marvin-cli/MANIFEST.in | 3 +
marvin-cli/Makefile | 53 ++
marvin-cli/README.md | 67 +++
marvin-cli/bin/marvin | 137 +++++
marvin-cli/marvin_cli/VERSION | 1 +
marvin-cli/marvin_cli/__init__.py | 16 +
marvin-cli/marvin_cli/communication/__init__.py | 16 +
.../marvin_cli/communication/remote_calls.py | 120 +++++
.../marvin_cli/communication/stubs/__init__.py | 16 +
.../marvin_cli/communication/stubs/daemon_pb2.py | 388 ++++++++++++++
.../communication/stubs/daemon_pb2_grpc.py | 129 +++++
marvin-cli/marvin_cli/management/__init__.py | 16 +
marvin-cli/marvin_cli/management/edit.py | 35 ++
marvin-cli/marvin_cli/management/engine.py | 202 ++++++++
marvin-cli/marvin_cli/management/notebook.py | 41 ++
.../management/template/cookiecutter.json | 10 +
.../{{cookiecutter.project_name}}/.bumpversion.cfg | 11 +
.../{{cookiecutter.project_name}}/.coveragerc | 22 +
.../{{cookiecutter.project_name}}/.gitignore | 16 +
.../{{cookiecutter.project_name}}/CHANGES.md | 5 +
.../template/{{cookiecutter.project_name}}/INSTALL | 1 +
.../template/{{cookiecutter.project_name}}/LICENSE | 0
.../{{cookiecutter.project_name}}/MANIFEST.in | 9 +
.../{{cookiecutter.project_name}}/Makefile | 57 +++
.../{{cookiecutter.project_name}}/README.md | 143 ++++++
.../docker/develop/daemon/Dockerfile | 82 +++
.../{{cookiecutter.project_name}}/docs.yaml | 567 +++++++++++++++++++++
.../{{cookiecutter.project_name}}/engine.messages | 3 +
.../{{cookiecutter.project_name}}/engine.metadata | 65 +++
.../{{cookiecutter.project_name}}/engine.params | 3 +
.../feedback.messages | 3 +
.../{{cookiecutter.project_name}}/marvin.ini | 3 +
.../notebooks/sample.ipynb | 113 ++++
.../{{cookiecutter.project_name}}/pytest.ini | 4 +
.../scripts/install_automl.sh | 35 ++
.../{{cookiecutter.project_name}}/setup.cfg | 0
.../{{cookiecutter.project_name}}/setup.py | 137 +++++
.../tests/conftest.py | 13 +
.../data_handler/test_acquisitor_and_cleaner.py | 18 +
.../tests/data_handler/test_training_preparator.py | 17 +
.../tests/prediction/test_feedback.py | 17 +
.../tests/prediction/test_prediction_preparator.py | 17 +
.../tests/prediction/test_predictor.py | 17 +
.../tests/training/test_metrics_evaluator.py | 17 +
.../tests/training/test_trainer.py | 18 +
.../template/{{cookiecutter.project_name}}/tox.ini | 9 +
.../{{cookiecutter.project_package}}/VERSION | 1 +
.../{{cookiecutter.project_package}}/__init__.py | 14 +
.../_compatibility.py | 19 +
.../{{cookiecutter.project_package}}/_logging.py | 78 +++
.../data_handler/__init__.py | 5 +
.../data_handler/acquisitor_and_cleaner.py | 34 ++
.../data_handler/training_preparator.py | 35 ++
.../prediction/__init__.py | 6 +
.../prediction/feedback.py | 44 ++
.../prediction/prediction_preparator.py | 30 ++
.../prediction/predictor.py | 30 ++
.../training/__init__.py | 5 +
.../training/metrics_evaluator.py | 34 ++
.../training/trainer.py | 34 ++
marvin-cli/marvin_cli/management/test.py | 54 ++
marvin-cli/marvin_cli/utils/__init__.py | 16 +
marvin-cli/marvin_cli/utils/config.py | 63 +++
marvin-cli/marvin_cli/utils/docker.py | 101 ++++
marvin-cli/marvin_cli/utils/git.py | 31 ++
marvin-cli/marvin_cli/utils/log.py | 93 ++++
marvin-cli/marvin_cli/utils/misc.py | 64 +++
marvin-cli/pytest.ini | 4 +
marvin-cli/setup.py | 111 ++++
.../tests/communication/test_remote_calls.py | 100 ++++
marvin-cli/tests/management/test_edit.py | 44 ++
marvin-cli/tests/management/test_engine.py | 34 ++
marvin-cli/tests/utils/test_config.py | 46 ++
marvin-cli/tests/utils/test_docker.py | 67 +++
marvin-cli/tests/utils/test_git.py | 38 ++
marvin-cli/tests/utils/test_misc.py | 68 +++
marvin-cli/tox.ini | 9 +
77 files changed, 3984 insertions(+)
diff --git a/marvin-cli/MANIFEST.in b/marvin-cli/MANIFEST.in
new file mode 100644
index 0000000..56d4d98
--- /dev/null
+++ b/marvin-cli/MANIFEST.in
@@ -0,0 +1,3 @@
+include README.md
+include marvin_cli/VERSION
+graft marvin_cli/management/template
\ No newline at end of file
diff --git a/marvin-cli/Makefile b/marvin-cli/Makefile
new file mode 100644
index 0000000..4f52365
--- /dev/null
+++ b/marvin-cli/Makefile
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.PHONY: help install test clean clean-build
+
+help:
+ @echo " install"
+ @echo " Install the python-daemon on system."
+ @echo " test"
+ @echo " Run unit tests."
+ @echo " clean"
+ @echo " Clean former builds"
+
+install:
+ pip install .
+
+test:
+ tox
+
+clean-pyc:
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
+ find . -name '__pycache__' -exec rm -rf {} +
+
+clean-build:
+ rm -f .prod
+ rm -rf *.egg-info
+ rm -rf .cache
+ rm -rf .eggs
+ rm -rf dist
+
+clean-reports:
+ rm -rf coverage_report/
+ rm -rf html-cov/
+ rm -f coverage.xml
+ rm -f .coverage
+ rm -rf .tox
+ rm -rf .pytest_cache
+
+clean: clean-pyc clean-build clean-reports
diff --git a/marvin-cli/README.md b/marvin-cli/README.md
new file mode 100644
index 0000000..592f8b6
--- /dev/null
+++ b/marvin-cli/README.md
@@ -0,0 +1,67 @@
+# Marvin CLI v0.0.5
+
+
+
+# Quick Start
+
+## Review
+
+**Marvin** is an open-source Artificial Intelligence platform that focuses on helping data scientists deliver meaningful solutions to complex problems. Supported by a standardized large-scale, language-agnostic architecture, Marvin simplifies the process of exploration and modeling.
+
+## Getting Started
+* [Installing Marvin (Ubuntu)](https://marvin.apache.org/marvin-platform-book/ch2_toolbox_installation/ubuntu/)
+* [Installing Marvin (MacOS)](https://marvin.apache.org/marvin-platform-book/ch2_toolbox_installation/mac/)
+* [Installing Marvin (Other OS) Vagrant](https://marvin.apache.org/marvin-platform-book/ch2_toolbox_installation/vagrant/)
+* [Creating a new engine](#creating-a-new-engine)
+* [Working in an existing engine](#working-in-an-existing-engine)
+* [Command line interface](#command-line-interface)
+* [Running an example engine](#running-a-example-engine)
+
+
+### Creating a new engine
+1. To create a new engine
+```
+marvin project-generate
+```
+Respond to the prompt and wait for the engine environment preparation to complete. Don't forget to start dev box before if you are using vagrant.
+
+2. Test the new engine
+```
+cd $MARVIN_HOME/engine
+marvin test
+```
+
+3. For more information
+```
+marvin --help
+```
+
+### Command line interface
+Usage: marvin [OPTIONS] COMMAND [ARGS]
+
+Options:
+```
+ --help #Show this command line interface and exit.
+```
+
+Commands:
+```
+ edit-config Edit configuration.
+ project-generate Generate engine project.
+ project-import Import engine project from archive file.
+ edit-metadata Edit engine.metadata.
+ engine-bumpversion Bump, commit and tag engine version.
+ engine-dryrun Run engines in a standalone way.
+ engine-grpcserver Run gRPC of given actions.
+ engine-httpserver Run executor HTTP server.
+ engine-logs Show daemon execution.
+ lab Run custom engine Jupyter Lab.
+ notebook Run custom engine Jupyter Notebook.
+ project-export Export engine project to a archive file.
+ test Run tests.
+ test-tdd Watch for changes to run tests automatically.
+ test-tox Run tests using Tox environment.
+```
+
+> Marvin is a project started at B2W Digital offices and released open source on September 2017.
+> The project is donated to Apache Software Foundation on August 2018.
diff --git a/marvin-cli/bin/marvin b/marvin-cli/bin/marvin
new file mode 100644
index 0000000..7ac2853
--- /dev/null
+++ b/marvin-cli/bin/marvin
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import click
+from marvin_cli.utils.misc import package_to_name
+from marvin_cli.utils.config import parse_ini
+from marvin_cli.utils.config import read_cli_conf, generate_default_conf
+from marvin_cli.utils.docker import search_engine_container
+from marvin_cli.utils.docker import search_engine_images
+from marvin_cli.utils.docker import create_engine_image
+from marvin_cli.utils.docker import create_daemon_container
+from marvin_cli.management.engine import cli as cli_engine
+from marvin_cli.management.notebook import cli as cli_notebook
+from marvin_cli.management.edit import cli as cli_edit
+from marvin_cli.management.test import cli as cli_test
+
+EXCLUDE_BY_TYPE = {
+ 'toolbox': ['engine-dryrun', 'engine-grpcserver', 'engine-httpserver', 'notebook',
+ 'lab', 'project-export', 'engine-logs', 'edit-metadata', 'test', 'test-tdd',
+ 'test-tox', 'engine-bumpversion'],
+ 'engine': ['project-import']
+}
+
+ENVIRONMENT_VARIABLES = {
+ 'MARVIN_HOME': "/home/{0}/marvin".format(os.environ['USER']),
+ 'MARVIN_DATA_PATH': "/home/{0}/marvin/data".format(os.environ['USER']),
+ 'MARVIN_LOG': "/home/{0}/marvin/data/.logs".format(os.environ['USER'])
+}
+
+def _define_environ():
+ for var in ENVIRONMENT_VARIABLES.keys():
+ if os.environ.get(var) is None:
+ os.environ[var] = ENVIRONMENT_VARIABLES[var]
+
+def _create_folders():
+ _exports_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
+ _conf_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf')
+ _artifacts_folder = os.path.join(os.environ['MARVIN_DATA_PATH'], '.artifacts')
+
+ _paths_list = [os.environ['MARVIN_HOME'], os.environ['MARVIN_DATA_PATH'],
+ os.environ['MARVIN_LOG'], _exports_folder, _conf_folder, _artifacts_folder]
+
+ for path in _paths_list:
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+def _initial_check_engine_mode(engine_package):
+ if not search_engine_images(engine_package):
+ if click.confirm('Do you want to create the engine docker image?', default=True):
+ create_engine_image(engine_package)
+ if not search_engine_container(engine_package):
+ if click.confirm('Do you want to create the engine docker container?', default=True):
+ create_daemon_container(engine_package, package_to_name(engine_package))
+
+def _read_config(engine_package):
+ filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
+ if not os.path.isfile(filepath):
+ conf = generate_default_conf()
+ else:
+ conf = read_cli_conf()
+ return conf
+
+
+sigh = """___ ___ _ _____ _ _____
+| \/ | (_) / __ \| | |_ _|
+| . . | __ _ _ ____ ___ _ __ | / \/| | | |
+| |\/| |/ _` | '__\ \ / / | '_ \ | | | | | |
+| | | | (_| | | \ V /| | | | | | \__/\| |_____| |_
+\_| |_/\__,_|_| \_/ |_|_| |_| \____/\_____/\___/\n\n"""
+
+ini_path = os.path.join(os.getcwd(), 'marvin.ini')
+
+config = None
+package_name = None
+exclude = None
+
+print(sigh)
+
+_define_environ()
+_create_folders()
+
+if os.path.exists(ini_path):
+ config = parse_ini(ini_path)
+ package_name = config['marvin_package']
+ exclude = EXCLUDE_BY_TYPE['engine']
+ print("[+] engine mode - {}".format(package_name))
+ _initial_check_engine_mode(package_name)
+ print("\n")
+else:
+ exclude = EXCLUDE_BY_TYPE['toolbox']
+ print("[+] toolbox mode\n")
+
+config = _read_config(package_name)
+
+@click.group('main')
+@click.pass_context
+def cli(ctx):
+ ctx.obj = {
+ 'package_name': package_name,
+ 'editor': config['editor'],
+ 'default_host': config['default_host'],
+ 'executor_url': config['executor_url']
+ }
+
+commands = {}
+commands.update(cli_engine.commands)
+commands.update(cli_notebook.commands)
+commands.update(cli_edit.commands)
+commands.update(cli_test.commands)
+
+for name, command in commands.items():
+ if name not in exclude:
+ cli.add_command(command, name=name)
+
+
+cli()
+
+
+
+
+
+
diff --git a/marvin-cli/marvin_cli/VERSION b/marvin-cli/marvin_cli/VERSION
new file mode 100644
index 0000000..fa3de58
--- /dev/null
+++ b/marvin-cli/marvin_cli/VERSION
@@ -0,0 +1 @@
+0.0.5
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/__init__.py b/marvin-cli/marvin_cli/__init__.py
new file mode 100644
index 0000000..77c2bbc
--- /dev/null
+++ b/marvin-cli/marvin_cli/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/communication/__init__.py b/marvin-cli/marvin_cli/communication/__init__.py
new file mode 100644
index 0000000..77c2bbc
--- /dev/null
+++ b/marvin-cli/marvin_cli/communication/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/communication/remote_calls.py b/marvin-cli/marvin_cli/communication/remote_calls.py
new file mode 100644
index 0000000..4ec433a
--- /dev/null
+++ b/marvin-cli/marvin_cli/communication/remote_calls.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import grpc
+import time
+from ..utils.log import get_logger
+from .stubs import daemon_pb2
+from .stubs import daemon_pb2_grpc
+
+logger = get_logger('communication')
+
+class RemoteError(Exception):
+ pass
+
+COMMANDS = {
+ 'DRYRUN': daemon_pb2.Command.CommandType.DRYRUN,
+ 'TEST': daemon_pb2.Command.CommandType.TEST,
+ 'GRPC': daemon_pb2.Command.CommandType.GRPC,
+ 'TDD': daemon_pb2.Command.CommandType.TDD,
+ 'TOX': daemon_pb2.Command.CommandType.TOX,
+ 'NOTEBOOK': daemon_pb2.Command.CommandType.NOTEBOOK,
+ 'LAB': daemon_pb2.Command.CommandType.LAB
+}
+
+class RemoteCalls:
+
+ stub = None
+
+ def __init__(self, host='localhost', port=50057):
+ channel = grpc.insecure_channel("{}:{}".format(host, port))
+ self.stub = daemon_pb2_grpc.CommandCallStub(channel)
+
+ def call_command(self, name, parameters):
+ call = daemon_pb2.Command(command=COMMANDS[name], parameters=parameters)
+ response = self.stub.callCommand(call)
+ if response.status == daemon_pb2.Status.NOK:
+ raise RemoteError("Error during {}.".format(name))
+ else:
+ logger.info("{} triggered!".format(name))
+
+ def stop_command(self, name):
+ call = daemon_pb2.Interruption()
+ response = self.stub.stopCommand(call)
+ if response.status == daemon_pb2.Status.NOK:
+ raise RemoteError("Error during stop {}.".format(name))
+ else:
+ logger.info("{} stopped!".format(name))
+
+ def run_dryrun(self, actions, profiling):
+ parameters = {
+ 'action': actions,
+ 'profiling': str(profiling)
+ }
+
+ self.call_command('DRYRUN', parameters)
+
+ def run_grpc(self, actions, max_workers, max_rpc_workers):
+ parameters = {
+ 'action': actions
+ }
+ self.call_command('GRPC', parameters)
+
+ def stop_grpc(self):
+ self.stop_command('GRPC')
+
+ def run_notebook(self, port, enable_security):
+ parameters = {
+ 'port': port,
+ 'enable_security': str(enable_security)
+ }
+ self.call_command('NOTEBOOK', parameters)
+
+ def run_lab(self, port, enable_security):
+ parameters = {
+ 'port': port,
+ 'enable_security': str(enable_security)
+ }
+ self.call_command('LAB', parameters)
+
+ def run_test(self, cov, no_capture, pdb, args):
+ parameters = {
+ 'cov': str(cov),
+ 'no_capture': str(no_capture),
+ 'pdb': str(pdb),
+ 'args': args
+ }
+
+ self.call_command('TEST', parameters)
+
+ def run_tdd(self, cov, no_capture, pdb, partial, args):
+ parameters = {
+ 'cov': str(cov),
+ 'no_capture': str(no_capture),
+ 'pdb': str(pdb),
+ 'partial': str(partial),
+ 'args': args
+ }
+
+ self.call_command('TDD', parameters)
+
+ def run_tox(self, args):
+ parameters = {
+ 'args': args
+ }
+
+ self.call_command('TOX', parameters)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/communication/stubs/__init__.py b/marvin-cli/marvin_cli/communication/stubs/__init__.py
new file mode 100644
index 0000000..77c2bbc
--- /dev/null
+++ b/marvin-cli/marvin_cli/communication/stubs/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py
new file mode 100644
index 0000000..b761e09
--- /dev/null
+++ b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2.py
@@ -0,0 +1,388 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: daemon.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+ name='daemon.proto',
+ package='',
+ syntax='proto3',
+ serialized_options=None,
+ serialized_pb=b'\n\x0c\x64\x61\x65mon.proto\"\xf3\x01\n\x07\x43ommand\x12%\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\x14.Command.CommandType\x12,\n\nparameters\x18\x02 \x03(\x0b\x32\x18.Command.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n\x0b\x43ommandType\x12\n\n\x06\x44RYRUN\x10\x00\x12\x08\n\x04TEST\x10\x01\x12\x07\n\x03TDD\x10\x02\x12\x07\n\x03TOX\x10\x03\x12\x0c\n\x08NOTEBOOK\x10\x04\x12\x07\n\x03LAB\x [...]
+)
+
+
+
+_COMMAND_COMMANDTYPE = _descriptor.EnumDescriptor(
+ name='CommandType',
+ full_name='Command.CommandType',
+ filename=None,
+ file=DESCRIPTOR,
+ values=[
+ _descriptor.EnumValueDescriptor(
+ name='DRYRUN', index=0, number=0,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='TEST', index=1, number=1,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='TDD', index=2, number=2,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='TOX', index=3, number=3,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='NOTEBOOK', index=4, number=4,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='LAB', index=5, number=5,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='GRPC', index=6, number=6,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='HTTP', index=7, number=7,
+ serialized_options=None,
+ type=None),
+ ],
+ containing_type=None,
+ serialized_options=None,
+ serialized_start=164,
+ serialized_end=260,
+)
+_sym_db.RegisterEnumDescriptor(_COMMAND_COMMANDTYPE)
+
+_STATUS_STATUSTYPE = _descriptor.EnumDescriptor(
+ name='StatusType',
+ full_name='Status.StatusType',
+ filename=None,
+ file=DESCRIPTOR,
+ values=[
+ _descriptor.EnumValueDescriptor(
+ name='OK', index=0, number=0,
+ serialized_options=None,
+ type=None),
+ _descriptor.EnumValueDescriptor(
+ name='NOK', index=1, number=1,
+ serialized_options=None,
+ type=None),
+ ],
+ containing_type=None,
+ serialized_options=None,
+ serialized_start=335,
+ serialized_end=364,
+)
+_sym_db.RegisterEnumDescriptor(_STATUS_STATUSTYPE)
+
+
+_COMMAND_PARAMETERSENTRY = _descriptor.Descriptor(
+ name='ParametersEntry',
+ full_name='Command.ParametersEntry',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='key', full_name='Command.ParametersEntry.key', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='value', full_name='Command.ParametersEntry.value', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=b'8\001',
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=113,
+ serialized_end=162,
+)
+
+_COMMAND = _descriptor.Descriptor(
+ name='Command',
+ full_name='Command',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='command', full_name='Command.command', index=0,
+ number=1, type=14, cpp_type=8, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='parameters', full_name='Command.parameters', index=1,
+ number=2, type=11, cpp_type=10, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[_COMMAND_PARAMETERSENTRY, ],
+ enum_types=[
+ _COMMAND_COMMANDTYPE,
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=17,
+ serialized_end=260,
+)
+
+
+_INTERRUPTION = _descriptor.Descriptor(
+ name='Interruption',
+ full_name='Interruption',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=262,
+ serialized_end=276,
+)
+
+
+_REQUEST = _descriptor.Descriptor(
+ name='Request',
+ full_name='Request',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=278,
+ serialized_end=287,
+)
+
+
+_STATUS = _descriptor.Descriptor(
+ name='Status',
+ full_name='Status',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='status', full_name='Status.status', index=0,
+ number=1, type=14, cpp_type=8, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ _STATUS_STATUSTYPE,
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=289,
+ serialized_end=364,
+)
+
+
+_STATE = _descriptor.Descriptor(
+ name='State',
+ full_name='State',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='engine_name', full_name='State.engine_name', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='command', full_name='State.command', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=366,
+ serialized_end=411,
+)
+
+_COMMAND_PARAMETERSENTRY.containing_type = _COMMAND
+_COMMAND.fields_by_name['command'].enum_type = _COMMAND_COMMANDTYPE
+_COMMAND.fields_by_name['parameters'].message_type = _COMMAND_PARAMETERSENTRY
+_COMMAND_COMMANDTYPE.containing_type = _COMMAND
+_STATUS.fields_by_name['status'].enum_type = _STATUS_STATUSTYPE
+_STATUS_STATUSTYPE.containing_type = _STATUS
+DESCRIPTOR.message_types_by_name['Command'] = _COMMAND
+DESCRIPTOR.message_types_by_name['Interruption'] = _INTERRUPTION
+DESCRIPTOR.message_types_by_name['Request'] = _REQUEST
+DESCRIPTOR.message_types_by_name['Status'] = _STATUS
+DESCRIPTOR.message_types_by_name['State'] = _STATE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Command = _reflection.GeneratedProtocolMessageType('Command', (_message.Message,), {
+
+ 'ParametersEntry' : _reflection.GeneratedProtocolMessageType('ParametersEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _COMMAND_PARAMETERSENTRY,
+ '__module__' : 'daemon_pb2'
+ # @@protoc_insertion_point(class_scope:Command.ParametersEntry)
+ })
+ ,
+ 'DESCRIPTOR' : _COMMAND,
+ '__module__' : 'daemon_pb2'
+ # @@protoc_insertion_point(class_scope:Command)
+ })
+_sym_db.RegisterMessage(Command)
+_sym_db.RegisterMessage(Command.ParametersEntry)
+
+Interruption = _reflection.GeneratedProtocolMessageType('Interruption', (_message.Message,), {
+ 'DESCRIPTOR' : _INTERRUPTION,
+ '__module__' : 'daemon_pb2'
+ # @@protoc_insertion_point(class_scope:Interruption)
+ })
+_sym_db.RegisterMessage(Interruption)
+
+Request = _reflection.GeneratedProtocolMessageType('Request', (_message.Message,), {
+ 'DESCRIPTOR' : _REQUEST,
+ '__module__' : 'daemon_pb2'
+ # @@protoc_insertion_point(class_scope:Request)
+ })
+_sym_db.RegisterMessage(Request)
+
+Status = _reflection.GeneratedProtocolMessageType('Status', (_message.Message,), {
+ 'DESCRIPTOR' : _STATUS,
+ '__module__' : 'daemon_pb2'
+ # @@protoc_insertion_point(class_scope:Status)
+ })
+_sym_db.RegisterMessage(Status)
+
+State = _reflection.GeneratedProtocolMessageType('State', (_message.Message,), {
+ 'DESCRIPTOR' : _STATE,
+ '__module__' : 'daemon_pb2'
+ # @@protoc_insertion_point(class_scope:State)
+ })
+_sym_db.RegisterMessage(State)
+
+
+_COMMAND_PARAMETERSENTRY._options = None
+
+_COMMANDCALL = _descriptor.ServiceDescriptor(
+ name='CommandCall',
+ full_name='CommandCall',
+ file=DESCRIPTOR,
+ index=0,
+ serialized_options=None,
+ serialized_start=413,
+ serialized_end=535,
+ methods=[
+ _descriptor.MethodDescriptor(
+ name='callCommand',
+ full_name='CommandCall.callCommand',
+ index=0,
+ containing_service=None,
+ input_type=_COMMAND,
+ output_type=_STATUS,
+ serialized_options=None,
+ ),
+ _descriptor.MethodDescriptor(
+ name='stopCommand',
+ full_name='CommandCall.stopCommand',
+ index=1,
+ containing_service=None,
+ input_type=_INTERRUPTION,
+ output_type=_STATUS,
+ serialized_options=None,
+ ),
+ _descriptor.MethodDescriptor(
+ name='getState',
+ full_name='CommandCall.getState',
+ index=2,
+ containing_service=None,
+ input_type=_REQUEST,
+ output_type=_STATE,
+ serialized_options=None,
+ ),
+])
+_sym_db.RegisterServiceDescriptor(_COMMANDCALL)
+
+DESCRIPTOR.services_by_name['CommandCall'] = _COMMANDCALL
+
+# @@protoc_insertion_point(module_scope)
diff --git a/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py
new file mode 100644
index 0000000..4f9e906
--- /dev/null
+++ b/marvin-cli/marvin_cli/communication/stubs/daemon_pb2_grpc.py
@@ -0,0 +1,129 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+#do this relative import to get it right
+from . import daemon_pb2 as daemon__pb2
+
+
+class CommandCallStub(object):
+ """Missing associated documentation comment in .proto file"""
+
+ def __init__(self, channel):
+ """Constructor.
+
+ Args:
+ channel: A grpc.Channel.
+ """
+ self.callCommand = channel.unary_unary(
+ '/CommandCall/callCommand',
+ request_serializer=daemon__pb2.Command.SerializeToString,
+ response_deserializer=daemon__pb2.Status.FromString,
+ )
+ self.stopCommand = channel.unary_unary(
+ '/CommandCall/stopCommand',
+ request_serializer=daemon__pb2.Interruption.SerializeToString,
+ response_deserializer=daemon__pb2.Status.FromString,
+ )
+ self.getState = channel.unary_unary(
+ '/CommandCall/getState',
+ request_serializer=daemon__pb2.Request.SerializeToString,
+ response_deserializer=daemon__pb2.State.FromString,
+ )
+
+
+class CommandCallServicer(object):
+ """Missing associated documentation comment in .proto file"""
+
+ def callCommand(self, request, context):
+ """Missing associated documentation comment in .proto file"""
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+ context.set_details('Method not implemented!')
+ raise NotImplementedError('Method not implemented!')
+
+ def stopCommand(self, request, context):
+ """Missing associated documentation comment in .proto file"""
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+ context.set_details('Method not implemented!')
+ raise NotImplementedError('Method not implemented!')
+
+ def getState(self, request, context):
+ """Missing associated documentation comment in .proto file"""
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+ context.set_details('Method not implemented!')
+ raise NotImplementedError('Method not implemented!')
+
+
+def add_CommandCallServicer_to_server(servicer, server):
+ rpc_method_handlers = {
+ 'callCommand': grpc.unary_unary_rpc_method_handler(
+ servicer.callCommand,
+ request_deserializer=daemon__pb2.Command.FromString,
+ response_serializer=daemon__pb2.Status.SerializeToString,
+ ),
+ 'stopCommand': grpc.unary_unary_rpc_method_handler(
+ servicer.stopCommand,
+ request_deserializer=daemon__pb2.Interruption.FromString,
+ response_serializer=daemon__pb2.Status.SerializeToString,
+ ),
+ 'getState': grpc.unary_unary_rpc_method_handler(
+ servicer.getState,
+ request_deserializer=daemon__pb2.Request.FromString,
+ response_serializer=daemon__pb2.State.SerializeToString,
+ ),
+ }
+ generic_handler = grpc.method_handlers_generic_handler(
+ 'CommandCall', rpc_method_handlers)
+ server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class CommandCall(object):
+ """Missing associated documentation comment in .proto file"""
+
+ @staticmethod
+ def callCommand(request,
+ target,
+ options=(),
+ channel_credentials=None,
+ call_credentials=None,
+ compression=None,
+ wait_for_ready=None,
+ timeout=None,
+ metadata=None):
+ return grpc.experimental.unary_unary(request, target, '/CommandCall/callCommand',
+ daemon__pb2.Command.SerializeToString,
+ daemon__pb2.Status.FromString,
+ options, channel_credentials,
+ call_credentials, compression, wait_for_ready, timeout, metadata)
+
+ @staticmethod
+ def stopCommand(request,
+ target,
+ options=(),
+ channel_credentials=None,
+ call_credentials=None,
+ compression=None,
+ wait_for_ready=None,
+ timeout=None,
+ metadata=None):
+ return grpc.experimental.unary_unary(request, target, '/CommandCall/stopCommand',
+ daemon__pb2.Interruption.SerializeToString,
+ daemon__pb2.Status.FromString,
+ options, channel_credentials,
+ call_credentials, compression, wait_for_ready, timeout, metadata)
+
+ @staticmethod
+ def getState(request,
+ target,
+ options=(),
+ channel_credentials=None,
+ call_credentials=None,
+ compression=None,
+ wait_for_ready=None,
+ timeout=None,
+ metadata=None):
+ return grpc.experimental.unary_unary(request, target, '/CommandCall/getState',
+ daemon__pb2.Request.SerializeToString,
+ daemon__pb2.State.FromString,
+ options, channel_credentials,
+ call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/marvin-cli/marvin_cli/management/__init__.py b/marvin-cli/marvin_cli/management/__init__.py
new file mode 100644
index 0000000..77c2bbc
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/edit.py b/marvin-cli/marvin_cli/management/edit.py
new file mode 100644
index 0000000..37b24b6
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/edit.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import click
+
+@click.group("edit")
+def cli():
+ pass
+
+@cli.command("edit-config", help="Edit configuration.")
+@click.pass_context
+def config(ctx):
+ filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
+ os.system(ctx.obj['editor'] + ' ' + filepath)
+
+@cli.command("edit-metadata", help="Edit engine.metadata.")
+@click.pass_context
+def metadata(ctx):
+ filepath = "engine.metadata"
+ os.system(ctx.obj['editor'] + ' ' + filepath)
diff --git a/marvin-cli/marvin_cli/management/engine.py b/marvin-cli/marvin_cli/management/engine.py
new file mode 100644
index 0000000..e813aa8
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/engine.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import os
+import sys
+import time
+import wget
+import subprocess
+import pathlib
+from cookiecutter.main import cookiecutter
+from ..communication.remote_calls import RemoteCalls
+from ..utils.misc import package_folder, extract_folder, get_version
+from ..utils.misc import call_logs, package_to_name, get_executor_path_or_download
+from ..utils.git import git_init, bump_version
+from ..utils.log import get_logger
+
+logger = get_logger('engine')
+
+@click.group("engine")
+def cli():
+ pass
+
+def _validate_project_name(name):
+ return ''.join(c for c in name if c.isalnum())
+
+TEMPLATE_FOLDER = os.path.join(pathlib.Path(__file__).parent.absolute(), 'template')
+
+@cli.command("project-generate", help="Generate engine project")
+@click.option('--name', '-n', prompt='Project name', help='Engine name')
+@click.option('--description', '-d', prompt='Project description', help='Engine description')
+@click.option('--url', '-u', prompt='Project URL', help='Engine URL Address', default='marvin.apache.org')
+@click.option('--maintainer', '-m', prompt='Maintainer', help='Engine Maintainer')
+@click.option('--email', '-e', prompt='Maintainer E-mail', help='Engine maintainer e-mail')
+@click.option('--dest', '-f', envvar='MARVIN_HOME', type=click.Path(exists=True), help='Root folder path for the creation')
+@click.option('--template', '-t', help='Base template for engine.', default=TEMPLATE_FOLDER, type=click.Path(exists=True))
+def generate(name, description, url, maintainer, email, dest, template):
+ _processed_name = _validate_project_name(name)
+
+ _extras_dir = {
+ 'project_name': _processed_name,
+ 'project_package': 'marvin_' + _processed_name.lower(),
+ 'project_url': url,
+ 'project_description': description,
+ 'maintainer_name': maintainer,
+ 'maintainer_email': email
+ }
+ print(TEMPLATE_FOLDER)
+ _init_dir = os.path.join(dest, _processed_name)
+ cookiecutter(template, output_dir=dest, extra_context=_extras_dir, no_input=True)
+ git_init(_init_dir)
+
+ logger.info("Engine {0} created".format(_processed_name))
+
+EXPORT_PATH = os.path.join(os.environ['MARVIN_DATA_PATH'], 'exports')
+
+@cli.command("project-export", help="Export engine project to a archive file.")
+@click.option('--dest', '-d', default=EXPORT_PATH, type=click.Path(exists=True), help='Output folder.')
+@click.pass_context
+def export(ctx, dest):
+ filename = os.path.join(dest, ctx.obj['package_name']
+ + '-' + get_version(ctx.obj['package_name']) + ".tar.gz")
+
+ package_folder(os.getcwd(), filename)
+
+@cli.command("project-import", help="Import engine project from archive file.")
+@click.option('--file', '-f', type=click.Path(exists=True), help='Compressed Engine file.')
+@click.option('--dest', '-d', envvar='MARVIN_HOME', type=click.Path(exists=True), help='Path to extract')
+def import_project(file, dest):
+ extract_folder(file, dest)
+
+@cli.command("engine-dryrun", help="Run engines in a standalone way.")
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option(
+ '--action',
+ '-a',
+ default='all',
+ type=click.Choice(['all', 'acquisitor', 'tpreparator', 'trainer', 'evaluator', 'ppreparator', 'predictor', 'feedback']),
+ help='Marvin engine action name')
+@click.option('--profiling', '-p', default=False, is_flag=True, help='Deterministic profiling of user code.')
+def dryrun(host, port, action, profiling):
+ rc = RemoteCalls(host, port)
+ rc.run_dryrun(action, profiling)
+
+@cli.command("engine-grpcserver", help="Run gRPC of given actions.")
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option(
+ '--action',
+ '-a',
+ default='all',
+ type=click.Choice(['all', 'acquisitor', 'tpreparator', 'trainer', 'evaluator', 'ppreparator', 'predictor', 'feedback']),
+ help='Marvin engine action name')
+@click.option('--max-workers', '-w', help='Max Workers', default=None)
+@click.option('--max-rpc-workers', '-rw', help='Max gRPC Workers', default=None)
+def grpc(host, port, action, max_workers, max_rpc_workers):
+ rc = RemoteCalls(host, port)
+ rc.run_grpc(action, max_workers, max_rpc_workers)
+ try:
+ while(True):
+ time.sleep(100)
+ except KeyboardInterrupt:
+ rc.stop_grpc()
+ logger.info("gRPC server terminated!")
+
+@cli.command("engine-logs", help="Show daemon execution.")
+@click.pass_context
+def docker_logs(ctx):
+ call_logs(ctx.obj['package_name'])
+
+@cli.command("engine-httpserver", help="Run executor HTTP server.")
+@click.option('--grpchost', '-gh', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--grpcport', '-gp', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--host', '-h', prompt='API host', help='REST API Host', default='localhost')
+@click.option('--port', '-p', prompt='API port', help='REST API Port', default='8000')
+@click.option('--protocol', '-pr', help='Marvin protocol to be loaded during initialization.', default='')
+@click.option(
+ '--action',
+ '-a',
+ default='all',
+ type=click.Choice(['all', 'acquisitor', 'tpreparator', 'trainer', 'evaluator', 'ppreparator', 'predictor', 'feedback']),
+ help='Marvin engine action name')
+@click.option('--max-workers', '-w', help='Max Workers', default=None)
+@click.option('--max-rpc-workers', '-rw', help='Max gRPC Workers', default=None)
+@click.option('--executor-path', '-e', help='Marvin engine executor jar path', type=click.Path(exists=True))
+@click.option('--extra-executor-parameters', '-jvm', help='Use to send extra JVM parameters to engine executor process')
+@click.pass_context
+def http(ctx, grpchost, grpcport, host, port, protocol, action, max_workers,
+ max_rpc_workers, executor_path, extra_executor_parameters):
+
+ rc = RemoteCalls(grpchost, grpcport)
+
+ try:
+ rc.run_grpc(action, max_workers, max_rpc_workers)
+ time.sleep(3)
+ except:
+ print("Could not start grpc server!")
+ sys.exit(1)
+
+ try:
+ if not executor_path:
+ executor_path = get_executor_path_or_download(ctx.obj['executor_url'])
+
+ command_list = ['java']
+ command_list.append('-DmarvinConfig.engineHome={}'.format(os.getcwd()))
+ command_list.append('-DmarvinConfig.ipAddress={}'.format(host))
+ command_list.append('-DmarvinConfig.port={}'.format(port))
+ command_list.append('-DmarvinConfig.protocol={}'.format(protocol))
+
+ if extra_executor_parameters:
+ command_list.append(extra_executor_parameters)
+
+ command_list.append('-jar')
+ command_list.append(executor_path)
+
+ print(command_list)
+
+ httpserver = subprocess.Popen(command_list)
+
+ except:
+ print("Could not start http server!")
+ rc.stop_grpc()
+ sys.exit(1)
+
+ try:
+ while True:
+ time.sleep(100)
+
+ except KeyboardInterrupt:
+ logger.info("Terminating http and grpc servers...")
+ rc.stop_grpc()
+ httpserver.terminate() if httpserver else None
+ logger.info("Http and grpc servers terminated!")
+ sys.exit(0)
+
+
+@cli.command('engine-bumpversion', help='Bump, commit and tag engine version.')
+@click.option('--verbose', is_flag=True)
+@click.option('--dry-run', is_flag=True)
+@click.option(
+ '--part',
+ '-p',
+ default='patch',
+ type=click.Choice(['major', 'minor', 'patch']),
+ help='The part of the version to increase.')
+def bumpversion(verbose, dry_run, part):
+ bump_version(part, verbose, dry_run)
diff --git a/marvin-cli/marvin_cli/management/notebook.py b/marvin-cli/marvin_cli/management/notebook.py
new file mode 100644
index 0000000..b3c5c4c
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/notebook.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+from ..communication.remote_calls import RemoteCalls
+
+@click.group("notebook")
+def cli():
+ pass
+
+@cli.command("notebook", help="Run custom engine Jupyter Notebook.")
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
+@click.option('--enable-security', '-s', default=False, is_flag=True, help='Enable notebook security.')
+def notebook(host, port, notebook_port, enable_security):
+ rc = RemoteCalls()
+ rc.run_notebook(notebook_port, enable_security)
+
+@cli.command("lab", help="Run custom engine Jupyter Lab.")
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--notebook-port', '-np', prompt='Notebook port', help='Notebook port', default='8888')
+@click.option('--enable-security', '-s', default=False, is_flag=True, help='Enable notebook security.')
+def lab(host, port, notebook_port, enable_security):
+ rc = RemoteCalls()
+ rc.run_lab(notebook_port, enable_security)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/cookiecutter.json b/marvin-cli/marvin_cli/management/template/cookiecutter.json
new file mode 100644
index 0000000..8554091
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/cookiecutter.json
@@ -0,0 +1,10 @@
+{
+ "project_name": "hello",
+ "project_package": "marvin_hello",
+ "project_description": "Marvin project",
+ "project_url": "marvin.apache.org",
+ "maintainer_name": "marvin",
+ "maintainer_email": "marvin@apache.org",
+ "daemon_version": "0.0.5",
+ "project_type": "python"
+}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg
new file mode 100644
index 0000000..68270c4
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.bumpversion.cfg
@@ -0,0 +1,11 @@
+[bumpversion]
+current_version = 0.0.1
+tag = True
+commit = True
+message = Bump version: [{now:%Y-%m-%d}] : {new_version}
+
+[bumpversion:file:marvin_testando123/VERSION]
+
+[bumpversion:file:README.md]
+
+[bumpversion:file:engine.metadata]
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc
new file mode 100644
index 0000000..6ebe78e
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.coveragerc
@@ -0,0 +1,22 @@
+[run]
+omit = tests/*
+branch = True
+
+[report]
+exclude_lines =
+ pragma: no cover
+
+ def __repr__
+ if self\.debug
+
+ raise AssertionError
+ raise NotImplementedError
+
+ if 0:
+ if __name__ == .__main__.:
+
+[html]
+directory = coverage_report
+
+[xml]
+output = coverage_report.xml
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore
new file mode 100644
index 0000000..05dd2a2
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/.gitignore
@@ -0,0 +1,16 @@
+.cache
+.eggs
+.tox
+.testmondata
+.coverage
+.coverage.*
+coverage_report.xml
+coverage_report
+*.egg
+*.egg-info
+*.pyc
+tests/__pycache__
+.DS_Store
+.packages
+.profiling
+notebooks/data
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md
new file mode 100644
index 0000000..4835f6f
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/CHANGES.md
@@ -0,0 +1,5 @@
+## Changes log
+
+### 0.0.1
+
+ - initial version
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL
new file mode 100644
index 0000000..fccdaf8
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/INSTALL
@@ -0,0 +1 @@
+REPLACE: Add here the detailed instructions to install this project
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/LICENSE
new file mode 100644
index 0000000..e69de29
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
new file mode 100644
index 0000000..c6c58b1
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/MANIFEST.in
@@ -0,0 +1,9 @@
+include CHANGES.md
+include INSTALL
+include LICENSE
+include MANIFEST.in
+include README.md
+include {{cookiecutter.project_package}}/VERSION
+recursive-include notebooks *
+prune notebooks/build
+recursive-include tests *
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile
new file mode 100644
index 0000000..53579d3
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/Makefile
@@ -0,0 +1,57 @@
+.PHONY: help marvin marvin-automl marvin-prod update clean-pyc clean-build clean-reports clean-deps clean
+
+help:
+ @echo " marvin"
+ @echo " Prepare project to be used as a marvin package."
+ @echo " marvin-prod"
+ @echo " Prepare project to be used in production environment."
+ @echo " marvin-automl"
+ @echo " Prepare project to be used as a marvin-automl package."
+ @echo " update"
+ @echo " Reinstall requirements and setup.py dependencies."
+ @echo " clean"
+ @echo " Remove all generated artifacts."
+ @echo " clean-pyc"
+ @echo " Remove python artifacts."
+ @echo " clean-build"
+ @echo " Remove build artifacts."
+ @echo " clean-reports"
+ @echo " Remove coverage reports."
+ @echo " clean-deps"
+ @echo " Remove marvin setup.py dependencies."
+
+marvin:
+ pip install -e ".[testing]"
+ marvin --help
+marvin-automl:
+ pip install -e ".[testing]"
+ bash scripts/install_automl.sh
+ marvin --help
+marvin-prod:
+ pip install .
+ marvin --help
+
+update:
+ pip install -e . -U
+
+clean-pyc:
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
+
+clean-build:
+ rm -rf *.egg-info
+ rm -rf .cache
+ rm -rf .eggs
+ rm -rf dist
+ rm -rf build
+
+clean-reports:
+ rm -rf coverage_report/
+ rm -f coverage.xml
+ rm -f .coverage
+
+clean-deps:
+ pip freeze | grep -v "^-e" | xargs pip uninstall -y
+
+clean: clean-build clean-pyc clean-reports clean-deps
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md
new file mode 100644
index 0000000..8144a24
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/README.md
@@ -0,0 +1,143 @@
+# {{cookiecutter.project_package}} v0.0.1
+
+## Overview
+
+{{cookiecutter.project_description}}
+
+
+## Requirements
+
+_REPLACE: Add here the list of requirements. For example:_
+
+ - Python 2.7
+ - Numpy 1.11.0 or higher
+
+
+## Installation
+
+Use the Marvin toolbox to provision, deploy and start the remote HTTP server.
+
+First, edit the `marvin.ini` file, setting the options within the
+`ssh_deployment` section:
+
+1. `host`: the host IP address or name where the engine should be deployed. You
+can enable multi-host deployment using `,` to separate hosts
+2. `port`: the SSH connection port
+3. `user`: the SSH connection username. Currently, only a single user is
+supported. This user should be capable of *passwordless sudo*, although it can
+use password for the SSH connection
+
+Next, ensure that the remotes servers are provisioned (all required software
+are installed):
+
+ marvin engine-deploy --provision
+
+Next, package your engine:
+
+ marvin engine-deploy --package
+
+This will create a compressed archive containing your engine code under the
+`.packages` directory.
+
+Next, deploy your engine to remotes servers:
+
+ marvin engine-deploy
+
+By default, a dependency clean will be executed at each deploy. You can skip it
+using:
+
+ marvin engine-deploy --skip-clean
+
+Next, you can start the HTTP server in the remotes servers:
+
+ marvin engine-httpserver-remote start
+
+You can check if the HTTP server is running:
+
+ marvin engine-httpserver-remote status
+
+And stop it:
+
+ marvin engine-httpserver-remote stop
+
+After starting, you can test it by making a HTTP request to any endpoint, like:
+
+ curl -v http://example.com/predictor/health
+
+Under the hood, this engine uses Fabric to define provisioning and deployment
+process. Check the `fabfile.py` for more information. You can add new tasks or
+edit existing ones to match your provisioning and deployment pipeline.
+
+## Development
+
+### Getting started
+
+First, create a new virtualenv
+
+```
+mkvirtualenv {{cookiecutter.project_package}}_env
+```
+
+Now install the development dependencies
+
+```
+make marvin
+```
+
+You are now ready to code.
+
+
+### Adding new dependencies
+
+It\`s very important. All development dependencies should be added to `setup.py`.
+
+### Running tests
+
+This project uses *[py.test](http://pytest.org/)* as test runner and *[Tox](https://tox.readthedocs.io)* to manage virtualenvs.
+
+To run all tests use the following command
+
+```
+marvin test
+```
+
+To run specific test
+
+```
+marvin test tests/test_file.py::TestClass::test_method
+```
+
+
+### Writting documentation
+
+The project documentation is written using *[Jupyter](http://jupyter.readthedocs.io/)* notebooks.
+You can start the notebook server from the command line by running the following command
+
+```
+marvin notebook
+```
+
+Use notebooks to demonstrate how to use the lib features. It can also be useful to show some use cases.
+
+
+### Bumping version
+
+```
+marvin pkg-bumpversion [patch|minor|major]
+git add . && git commit -m "Bump version"
+```
+
+
+### Tagging version
+
+```
+marvin pkg-createtag
+git push origin master --follow-tags
+```
+
+
+### Logging
+
+The default log level is set to _WARNING_. You can change the log level at runtime setting another value to one of the following environment variable: `{{cookiecutter.project_package}}_LOG_LEVEL` or `LOG_LEVEL`. The available values are _CRITICAL_, _ERROR_, _WARNING_, _INFO_ and _DEBUG_.
+
+Be careful using `LOG_LEVEL`, it may affect another lib.
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
new file mode 100644
index 0000000..8418ff2
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docker/develop/daemon/Dockerfile
@@ -0,0 +1,82 @@
+############################################################
+# DEVELOPMENT USE ONLY - USE THE RIGHT PROCEDURE TO DEPLOY #
+############################################################
+
+FROM marvin-daemon:python
+
+MAINTAINER {{cookiecutter.maintainer_name}}
+
+ENV SLEEP_MILLIS 0
+
+##############################################################
+# Define all environment variables to be used
+##############################################################
+
+ENV ENGINE={{cookiecutter.project_name}}
+ENV MARVIN_HOME=/opt/marvin
+ENV MARVIN_LOG=$MARVIN_HOME/log
+ENV MARVIN_DATA_PATH=/opt/marvin/data
+ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine
+ENV SPARK_HOME=/opt/spark
+ENV SPARK_CONF_DIR=$SPARK_HOME/conf
+ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR
+ENV YARN_CONF_DIR=$SPARK_CONF_DIR
+
+##############################################################
+# Create all folders needed
+##############################################################
+
+RUN mkdir -p $MARVIN_HOME && \
+ mkdir -p $MARVIN_DATA_PATH && \
+ mkdir -p $MARVIN_ENGINE_HOME && \
+ mkdir -p $MARVIN_LOG
+
+##############################################################
+# Install Apache Spark
+#
+# Uncomment if you are using spark, note that is needed the
+# spark configuration files to the think works correctly.
+##############################################################
+#
+# RUN wget -O /tmp/spark-2.1.1-bin-hadoop2.6.tgz \
+# https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \
+# tar -xf /tmp/spark-2.1.1-bin-hadoop2.6.tgz -C /opt/ && \
+# ln -s /opt/spark-2.1.1-bin-hadoop2.6 /opt/spark
+##############################################################
+
+RUN mkdir -p $SPARK_CONF_DIR
+
+##############################################################
+# INSTALL ENGINE FROM SOURCE #
+##############################################################
+
+COPY {{cookiecutter.project_package}}-0.0.1.tar.gz .
+RUN pip install {{cookiecutter.project_package}}-0.0.1.tar.gz
+
+##############################################################
+# CUSTOM ENGINE INSTALLATION PROCEDURE ABOVE #
+##############################################################
+
+RUN pip install --no-cache pandas \
+&& pip install --no-cache matplotlib \
+&& pip install --no-cache cython \
+&& pip install --no-cache numpy \
+&& pip install --no-cache scikit-learn \
+&& pip install --no-cache pytest \
+&& pip install --no-cache pytest-cov \
+&& pip install --no-cache pytest-watch
+
+##############################################################
+# Uninstalling unnecessary software and cleaning cache
+##############################################################
+
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
+
+##############################################################
+# Starts the daemon
+##############################################################
+
+EXPOSE 50051-50057
+EXPOSE 8888
+
+CMD /bin/bash -c "cd $MARVIN_ENGINE_HOME && marvin-daemon"
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml
new file mode 100644
index 0000000..d349095
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/docs.yaml
@@ -0,0 +1,567 @@
+openapi: "3.0.0"
+info:
+ version: 0.0.1
+ title: {{cookiecutter.project_package}} API Doc
+ contact:
+ name: maintainer {{cookiecutter.maintainer_name}}
+ email:
+ url: {{cookiecutter.project_url}}
+ license:
+ name: Apache License 2.0
+servers:
+ - url: http://localhost:8000
+ - url: http://0.0.0.0:8000
+tags:
+ - name: Docker
+ description: For Docker users, please use "make docker-build" and "make docker-run" commands in your engine virtualenv to start the server
+ - name: Acquisitor
+ description: Setup the initial_dataset with all cleaned data necessary to build your dataset in the next action
+ - name: Tpreparator
+ description: Setup the dataset with the transformed data that is compatible with the algorithm used to build the model in the next action
+ - name: Trainer
+ description: Setup the model with the result of the algorithm used to training
+ - name: Evaluator
+ description: Setup the metrics with the result of the algorithms used to test the model
+ - name: Predictor
+ description: Return the predicted value in a json parsable object format
+ - name: Feedback
+ description: Receive feedback message, user can manipulate this message for any use
+ - name: Pipeline
+ description: Perform all batch actions in your right order
+paths:
+ /acquisitor/health:
+ get:
+ summary: Get acquisitor's service health
+ operationId: getAcquisitorHealth
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor/status:
+ get:
+ summary: Get acquisitor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getAcquisitorStatus
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /acquisitor:
+ post:
+ summary: Run acquisitor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: acquisitor
+ tags:
+ - Acquisitor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/health:
+ get:
+ summary: Get trainer preparator's service health
+ operationId: getTPreparatortorHealth
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/status:
+ get:
+ summary: Get trainer preparator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTPreparatorStatus
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer preparator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTPreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /tpreparator:
+ post:
+ summary: Run trainer preparator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: tpreparator
+ tags:
+ - Tpreparator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/health:
+ get:
+ summary: Get trainer's service health
+ operationId: getTrainerHealth
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/status:
+ get:
+ summary: Get trainer's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getTrainerStatus
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer/reload?protocol=:
+ put:
+ summary: Reload artifact for trainer
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadTrainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /trainer:
+ post:
+ summary: Run trainer
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: trainer
+ tags:
+ - Trainer
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/health:
+ get:
+ summary: Get evaluator's service health
+ operationId: getEvaluatorHealth
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/status:
+ get:
+ summary: Get evaluator's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getEvaluatorStatus
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/metrics?protocol=:
+ get:
+ summary: Get metrics's value
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: Metrics protocol value
+ operationId: getMetrics
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator/reload?protocol=:
+ put:
+ summary: Reload artifact for evaluator
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: reloadEvaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /evaluator:
+ post:
+ summary: Run evaluator
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: evaluator
+ tags:
+ - Evaluator
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/health:
+ get:
+ summary: Get predictor's service health
+ operationId: getPredictorHealth
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/status:
+ get:
+ summary: Get predictor's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getPredictorStatus
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor/reload?protocol=:
+ put:
+ summary: Reload artifact for predictor
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadPredictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /predictor:
+ post:
+ summary: Run predictor
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: predictor
+ tags:
+ - Predictor
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/health:
+ get:
+ summary: Get feedback's service health
+ operationId: getFeedbackHealth
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/status:
+ get:
+ summary: Get feedback's service status
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ operationId: getFeedbackStatus
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback/reload?protocol=:
+ put:
+ summary: Reload artifact for feedback
+ parameters:
+ - in: query
+ name: protocol
+ schema:
+ type: string
+ required: true
+ description: The Protocol value generated from last action
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: reloadFeedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /feedback:
+ post:
+ summary: Run feedback
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: feedback
+ tags:
+ - Feedback
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
+ /pipeline:
+ post:
+ summary: Do all batch actions (from Acquisitor to Evaluator)
+ requestBody:
+ description: The default value for body is an empty json object
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ operationId: pipeline
+ tags:
+ - Pipeline
+ responses:
+ '200':
+ description: Result Message / Success
+ '400':
+ description: Bad Request / Illegal Argument / Missing Parameters
+ '500':
+ description: Internal Server Error / Timeout
+ '503':
+ description: Service Unavailable
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages
new file mode 100644
index 0000000..5ff7ec4
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.messages
@@ -0,0 +1,3 @@
+[{
+ "msg1": "Hello from marvin engine!"
+}]
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata
new file mode 100644
index 0000000..9072430
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.metadata
@@ -0,0 +1,65 @@
+{
+ "name": "{{cookiecutter.project_name}}",
+ "version": "v0.0.1",
+ "engineType": "python",
+ "artifactsRemotePath": "/tmp/marvin",
+ "artifactManagerType": "FS",
+ "onlineActionTimeout": 1000,
+ "metricsTimeout": 10000,
+ "healthCheckTimeout": 2000,
+ "reloadTimeout": 600000,
+ "batchActionTimeout": 600000,
+ "pipelineActions": ["acquisitor", "tpreparator", "trainer", "evaluator"],
+ "actions": [{
+ "name": "acquisitor",
+ "actionType": "batch",
+ "port": 50051,
+ "host": "localhost",
+ "artifactsToPersist": ["initialdataset"],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }, {
+ "name": "tpreparator",
+ "actionType": "batch",
+ "port": 50052,
+ "host": "localhost",
+ "artifactsToPersist": ["dataset"],
+ "artifactsToLoad": ["initialdataset"],
+ "pipeline": []
+ }, {
+ "name": "trainer",
+ "actionType": "batch",
+ "port": 50053,
+ "host": "localhost",
+ "artifactsToPersist": ["model"],
+ "artifactsToLoad": ["dataset"],
+ "pipeline": []
+ }, {
+ "name": "evaluator",
+ "actionType": "batch",
+ "port": 50054,
+ "host": "localhost",
+ "artifactsToPersist": ["metrics"],
+ "artifactsToLoad": ["dataset", "model"],
+ "pipeline": []
+ }, {
+ "name": "predictor",
+ "actionType": "online",
+ "port": 50055,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": ["model", "metrics"],
+ "pipeline": ["ppreparator"]
+ }, {
+ "name": "feedback",
+ "actionType": "online",
+ "port": 50056,
+ "host": "localhost",
+ "artifactsToPersist": [],
+ "artifactsToLoad": [],
+ "pipeline": []
+ }],
+ "bigquery_project": "xxx-project",
+ "bigquery_queries": [],
+ "bigquery_csvfiles": []
+}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params
new file mode 100644
index 0000000..410f586
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/engine.params
@@ -0,0 +1,3 @@
+{
+ "PARAM_1" : "VALUE_OF_PARAM_1"
+}
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages
new file mode 100644
index 0000000..5ff7ec4
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/feedback.messages
@@ -0,0 +1,3 @@
+[{
+ "msg1": "Hello from marvin engine!"
+}]
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini
new file mode 100644
index 0000000..7cd72ec
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/marvin.ini
@@ -0,0 +1,3 @@
+[marvin]
+package = {{cookiecutter.project_package}}
+type = {{cookiecutter.project_type}}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb
new file mode 100644
index 0000000..44bfcc3
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/notebooks/sample.ipynb
@@ -0,0 +1,113 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Documentation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Sample"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "marvin_cell": "acquisitor"
+ },
+ "outputs": [],
+ "source": [
+ "marvin_initial_dataset = {\n",
+ " 'marvin': 'tester'\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "marvin_cell": "tpreparator"
+ },
+ "outputs": [],
+ "source": [
+ "marvin_dataset = {\n",
+ " 'marvin': 'test'\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "marvin_cell": "trainer"
+ },
+ "outputs": [],
+ "source": [
+ "marvin_model = {\n",
+ " 'marvin': 'test'\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "marvin_cell": "evaluator"
+ },
+ "outputs": [],
+ "source": [
+ "marvin_metrics = {\n",
+ " 'marvin': 'test'\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "marvin_cell": "ppreparator"
+ },
+ "outputs": [],
+ "source": [
+ "input_message = \"marvin_test\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "marvin_cell": "predictor"
+ },
+ "outputs": [],
+ "source": [
+ "final_prediction = input_message"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini
new file mode 100644
index 0000000..cab8644
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+minversion = 2.0
+norecursedirs = .git .tox .eggs .cache *.egg build dist tmp*
+python_files = test*.py
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh
new file mode 100644
index 0000000..4bde530
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/scripts/install_automl.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+###############################################
+# Marvin AutoML installation Script #
+###############################################
+cmd=(dialog --title "Marvin AutoML" --separate-output --checklist "Select tools:" 22 76 16)
+options=(1 "auto-sklearn" off # any option can be set to default to "on"
+ 2 "h2o AutoML" off
+ 3 "TPOT" off)
+choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
+clear
+for choice in $choices
+do
+ case $choice in
+ 1)
+ echo "Instaling auto-sklearn..."
+ wget https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt \
+ | xargs -n 1 -L 1 pip install
+ pip install auto-sklearn
+ ;;
+ 2)
+ echo "Installing h2o..."
+ pip install requests
+ pip install tabulate
+ pip install scikit-learn
+ pip install http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/Python/h2o-3.26.0.3-py2.py3-none-any.whl
+ wget http://h2o-release.s3.amazonaws.com/h2o/rel-yau/3/h2o-3.26.0.3.zip
+ unzip h2o-3.26.0.3.zip
+ rm h2o-3.26.0.3.zip
+ ;;
+ 3)
+ echo "Installing TPOT..."
+ pip install tpot
+ ;;
+ esac
+done
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.cfg
new file mode 100644
index 0000000..e69de29
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py
new file mode 100644
index 0000000..3f380f3
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/setup.py
@@ -0,0 +1,137 @@
+from __future__ import print_function
+
+import os
+import shutil
+from os.path import dirname, join
+from setuptools import setup, find_packages
+from setuptools.command.test import test as TestCommand
+from setuptools.command.develop import develop as _develop
+from setuptools.command.install import install as _install
+
+
+REQUIREMENTS_TESTS = [
+ 'pytest>=2.6.4',
+ 'pytest-cov>=1.8.1',
+ 'mock>=2.0.0',
+ 'tox-current-env',
+ 'tox>=2.2.0',
+]
+
+def _get_version():
+ """Return the project version from VERSION file."""
+
+ with open(join(dirname(__file__), '{{cookiecutter.project_package}}/VERSION'), 'rb') as f:
+ version = f.read().decode('ascii').strip()
+ return version
+
+
+def _hooks(dir):
+ _install_notebook_extension()
+
+def _install_notebook_extension():
+ import marvin_python_daemon as daemon
+
+ install_command = [
+ "jupyter",
+ "nbextension",
+ "install",
+ os.path.join(daemon.__path__[0], 'extras', 'notebook_extensions', 'main.js'),
+ "--destination",
+ "marvin.js",
+ "--sys-prefix",
+ "--overwrite"
+ ]
+
+ os.system(' '.join(install_command))
+
+ enable_command = [
+ "jupyter",
+ "nbextension",
+ "enable",
+ "marvin",
+ "--sys-prefix"
+ ]
+
+ os.system(' '.join(enable_command))
+
+
+class develop(_develop):
+ def run(self):
+ _develop.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running develop preparation task")
+
+
+class install(_install):
+ def run(self):
+ _install.run(self)
+ self.execute(_hooks, (self.install_lib,), msg="Running install preparation task")
+
+
+class Tox(TestCommand):
+ """Run the test cases using TOX command."""
+
+ user_options = [('tox-args=', 'a', "Arguments to pass to tox")]
+
+ def initialize_options(self):
+ TestCommand.initialize_options(self)
+ self.tox_args = None
+
+ def finalize_options(self):
+ TestCommand.finalize_options(self)
+ self.test_args = []
+ self.test_suite = True
+
+ def run_tests(self):
+ # Import here, cause outside the eggs aren't loaded
+ import tox
+ import shlex
+ import sys
+ args = self.tox_args
+ if args:
+ args = shlex.split(self.tox_args)
+ else:
+ # Run all tests by default
+ args = ['-c', join(dirname(__file__), 'tox.ini'), 'tests']
+ errno = tox.cmdline(args=args)
+ sys.exit(errno)
+
+
+setup(
+ name='{{cookiecutter.project_package}}',
+ version=_get_version(),
+ url='{{cookiecutter.project_url}}',
+ description='{{cookiecutter.project_description}}',
+ long_description=open(join(dirname(__file__), 'README.md')).read(),
+ author='{{cookiecutter.maintainer_name}}',
+ maintainer='{{cookiecutter.maintainer_name}}',
+ maintainer_email='{{cookiecutter.maintainer_name}}',
+ packages=find_packages(exclude=('tests', 'tests.*')),
+ include_package_data=True,
+ zip_safe=False,
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Intended Audience :: Developers',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ ],
+ install_requires=[
+ 'scikit-learn>=0.18.2',
+ 'scipy>=0.19.1',
+ 'numpy>=1.13.1',
+ 'pandas>=0.20.3',
+ 'matplotlib>=2.0.2',
+ 'marvin-python-daemon>={{cookiecutter.daemon_version}}',
+ ],
+ tests_require=REQUIREMENTS_TESTS,
+ extras_require={
+ 'testing': REQUIREMENTS_TESTS,
+ },
+ cmdclass={
+ 'test': Tox, 'develop': develop, 'install': install
+ },
+)
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py
new file mode 100644
index 0000000..903cfc8
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/conftest.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os
+import pytest
+
+os.environ['TESTING'] = 'True'
+
+
+@pytest.fixture
+def mocked_params():
+ return {'params': 1}
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py
new file mode 100644
index 0000000..fdda106
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_acquisitor_and_cleaner.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.data_handler import AcquisitorAndCleaner
+
+
+class TestAcquisitorAndCleaner:
+ def test_execute(self, mocked_params):
+ ac = AcquisitorAndCleaner()
+ ac.execute(params=mocked_params)
+ assert not ac._params
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py
new file mode 100644
index 0000000..7dc8e6a
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/data_handler/test_training_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.data_handler import TrainingPreparator
+
+
+class TestTrainingPreparator:
+ def test_execute(self, mocked_params):
+ ac = TrainingPreparator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py
new file mode 100644
index 0000000..697a28d
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_feedback.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.prediction import Feedback
+
+
+class TestFeedback:
+ def test_execute(self, mocked_params):
+ fb = Feedback()
+ fb.execute(input_message="fake message", params=mocked_params)
+ assert not fb._params
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py
new file mode 100644
index 0000000..091c46f
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_prediction_preparator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.prediction import PredictionPreparator
+
+
+class TestPredictionPreparator:
+ def test_execute(self, mocked_params):
+ ac = PredictionPreparator()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py
new file mode 100644
index 0000000..f1ce576
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/prediction/test_predictor.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.prediction import Predictor
+
+
+class TestPredictor:
+ def test_execute(self, mocked_params):
+ ac = Predictor()
+ ac.execute(input_message="fake message", params=mocked_params)
+ assert not ac._params
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py
new file mode 100644
index 0000000..3c18e41
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_metrics_evaluator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.training import MetricsEvaluator
+
+
+class TestMetricsEvaluator:
+ def test_execute(self, mocked_params):
+ ac = MetricsEvaluator()
+ ac.execute(params=mocked_params)
+ assert not ac._params
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py
new file mode 100644
index 0000000..3f2c99a
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tests/training/test_trainer.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+try:
+ import mock
+
+except ImportError:
+ import unittest.mock as mock
+
+from {{cookiecutter.project_package}}.training import Trainer
+
+
+class TestTrainer:
+ def test_execute(self, mocked_params):
+ ac = Trainer()
+ ac.execute(params=mocked_params)
+ assert not ac._params
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini
new file mode 100644
index 0000000..42a8cbb
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/tox.ini
@@ -0,0 +1,9 @@
+[tox]
+envlist = py37
+
+[testenv]
+deps=pytest
+ pytest-cov
+ mock
+passenv=MARVIN_LOG MARVIN_DATA_PATH
+commands=python -m pytest --cov='.' --cov-report html --cov-report xml tests/
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION
new file mode 100644
index 0000000..8acdd82
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/VERSION
@@ -0,0 +1 @@
+0.0.1
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py
new file mode 100644
index 0000000..a45c1f1
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/__init__.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import os.path
+
+from .data_handler import *
+from .prediction import *
+from .training import *
+
+
+# Get package version number from "VERSION" file
+with open(os.path.join(os.path.dirname(__file__), 'VERSION'), 'rb') as f:
+ __version__ = f.read().decode('ascii').strip()
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py
new file mode 100644
index 0000000..4ce10fe
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_compatibility.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Compatibility module.
+
+Import this module to help to write code compatible with Python 2 and 3.
+"""
+
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import six
+
+__all__ = ['six']
+
+# Add here any code that have to differentiate between python 2 and 3.
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py
new file mode 100644
index 0000000..225b8e7
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/_logging.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Custom logging module.
+
+This module is responsible to manage log messages and log file.
+"""
+
+import sys
+import os
+import os.path
+import logging
+
+DEFAULT_LOG_LEVEL = logging.INFO
+DEFAULT_LOG_DIR = '/tmp'
+
+
+class Logger(logging.getLoggerClass()):
+ """Custom logger class.
+
+ Use this class to customize the logger behavior or to intercept the
+ messages.
+ """
+ def error(self, msg, *args, **kwargs):
+ # Add here code to intercept the project error messages
+ super(Logger, self).error(msg, *args, **kwargs)
+
+ def critical(self, msg, *args, **kwargs):
+ # Add here code to intercept the project critical messages
+ super(Logger, self).critical(msg, *args, **kwargs)
+
+
+logging.setLoggerClass(Logger)
+
+
+def get_logger(name, namespace='marvin_testando123',
+ log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ """Build a logger that outputs to a file and to the console,"""
+
+ log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
+ os.getenv('LOG_LEVEL', log_level))
+ log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
+ os.getenv('LOG_DIR', log_dir))
+
+ logger = logging.getLogger('{}.{}'.format(namespace, name))
+ logger.setLevel(log_level)
+
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ # Create a console stream handler
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(log_level)
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+ try:
+ if log_dir:
+ log_path = os.path.abspath(log_dir)
+ log_filename = '{name}.{pid}.log'.format(
+ name=namespace, pid=os.getpid())
+
+ file_path = str(os.path.join(log_path, log_filename))
+
+ if not os.path.exists(log_path):
+ os.makedirs(log_path, mode=774)
+
+ # Create a file handler
+ file_handler = logging.FileHandler(file_path)
+ file_handler.setLevel(log_level)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ except OSError as e:
+ logger.error('Could not create log file {file}: {error}'.format(
+ file=file_path, error=e.strerror))
+
+ return logger
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py
new file mode 100644
index 0000000..6707e49
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .acquisitor_and_cleaner import AcquisitorAndCleaner
+from .training_preparator import TrainingPreparator
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py
new file mode 100644
index 0000000..edf6518
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/acquisitor_and_cleaner.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""AcquisitorAndCleaner engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBaseDataHandler
+
+__all__ = ['AcquisitorAndCleaner']
+
+
+logger = get_logger('acquisitor_and_cleaner')
+
+
+class AcquisitorAndCleaner(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(AcquisitorAndCleaner, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ """
+ Setup the initial_dataset with all cleaned data necessary to build your dataset in the next action.
+
+ Eg.
+
+ self.marvin_initial_dataset = {...}
+ """
+ self.marvin_initial_dataset = {}
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py
new file mode 100644
index 0000000..f1c5862
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/data_handler/training_preparator.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""TrainingPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBaseDataHandler
+
+__all__ = ['TrainingPreparator']
+
+
+logger = get_logger('training_preparator')
+
+
+class TrainingPreparator(EngineBaseDataHandler):
+
+ def __init__(self, **kwargs):
+ super(TrainingPreparator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ """
+ Setup the dataset with the transformed data that is compatible with the algorithm used to build the model in the next action.
+ Use the self.initial_dataset prepared in the last action as source of data.
+
+ Eg.
+
+ self.marvin_dataset = {...}
+ """
+ self.marvin_dataset = {}
+
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py
new file mode 100644
index 0000000..aa6291d
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/__init__.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .prediction_preparator import PredictionPreparator
+from .predictor import Predictor
+from .feedback import Feedback
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py
new file mode 100644
index 0000000..3d64690
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/feedback.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2019] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Feedback engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBasePrediction
+
+__all__ = ['Feedback']
+
+
+logger = get_logger('feedback')
+
+
+class Feedback(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Feedback, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ """
+ Receive feedback message, user can manipulate this message for any use.
+ Return "Done" to signal that the message is received and processed.
+ """
+ return {"message": "Done"}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py
new file mode 100644
index 0000000..f6c6944
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/prediction_preparator.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""PredictionPreparator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBasePrediction
+
+__all__ = ['PredictionPreparator']
+
+
+logger = get_logger('prediction_preparator')
+
+
+class PredictionPreparator(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(PredictionPreparator, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ """
+ Return a prepared input_message compatible to the predict algorithm used by the model.
+ Use the self.model and self.metrics objects if necessary.
+ """
+ return input_message
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py
new file mode 100644
index 0000000..573efa0
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/prediction/predictor.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Predictor engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBasePrediction
+
+__all__ = ['Predictor']
+
+
+logger = get_logger('predictor')
+
+
+class Predictor(EngineBasePrediction):
+
+ def __init__(self, **kwargs):
+ super(Predictor, self).__init__(**kwargs)
+
+ def execute(self, input_message, params, **kwargs):
+ """
+ Return the predicted value in a json parsable object format.
+ Use the self.model and self.metrics objects if necessary.
+ """
+ return {"predicted_value": "mocked_return"}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py
new file mode 100644
index 0000000..e1723b7
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+from .metrics_evaluator import MetricsEvaluator
+from .trainer import Trainer
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py
new file mode 100644
index 0000000..6cacd78
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/metrics_evaluator.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""MetricsEvaluator engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBaseTraining
+
+__all__ = ['MetricsEvaluator']
+
+
+logger = get_logger('metrics_evaluator')
+
+
+class MetricsEvaluator(EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(MetricsEvaluator, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ """
+ Setup the metrics with the result of the algorithms used to test the model.
+ Use the self.dataset and self.model prepared in the last actions.
+
+ Eg.
+
+ self.marvin_metrics = {...}
+ """
+ self.marvin_metrics = {}
diff --git a/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py
new file mode 100644
index 0000000..dc25e44
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/template/{{cookiecutter.project_name}}/{{cookiecutter.project_package}}/training/trainer.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Trainer engine action.
+
+Use this module to add the project main code.
+"""
+
+from .._compatibility import six
+from .._logging import get_logger
+
+from marvin_python_daemon.engine_base import EngineBaseTraining
+
+__all__ = ['Trainer']
+
+
+logger = get_logger('trainer')
+
+
+class Trainer(EngineBaseTraining):
+
+ def __init__(self, **kwargs):
+ super(Trainer, self).__init__(**kwargs)
+
+ def execute(self, params, **kwargs):
+ """
+ Setup the model with the result of the algorithm used to training.
+ Use the self.dataset prepared in the last action as source of data.
+
+ Eg.
+
+ self.marvin_model = {...}
+ """
+ self.marvin_model = {}
diff --git a/marvin-cli/marvin_cli/management/test.py b/marvin-cli/marvin_cli/management/test.py
new file mode 100644
index 0000000..abd3dfb
--- /dev/null
+++ b/marvin-cli/marvin_cli/management/test.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+from ..communication.remote_calls import RemoteCalls
+
+@click.group("test")
+def cli():
+ pass
+
+@cli.command('test', help='Run tests.')
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--cov/--no-cov', default=True)
+@click.option('--no-capture', is_flag=True)
+@click.option('--pdb', is_flag=True)
+@click.argument('args', default='')
+def test(host, port, cov, no_capture, pdb, args):
+ rc = RemoteCalls(host, port)
+ rc.run_test(cov, no_capture, pdb, args)
+
+@cli.command('test-tox', help='Run tests using Tox environment.')
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.argument('args', default='--current-env')
+def tox(host, port, args):
+ rc = RemoteCalls(host, port)
+ rc.run_tox(args)
+
+@cli.command('test-tdd', help='Watch for changes to run tests automatically.')
+@click.option('--host', '-h', prompt='gRPC host', help='gRPC Host Address', default='localhost')
+@click.option('--port', '-p', prompt='gRPC port', help='gRPC Port', default='50057')
+@click.option('--cov/--no-cov', default=False)
+@click.option('--no-capture', is_flag=True)
+@click.option('--pdb', is_flag=True)
+@click.option('--partial', is_flag=True)
+@click.argument('args', default='')
+def tdd(host, port, cov, no_capture, pdb, partial, args):
+ rc = RemoteCalls(host, port)
+ rc.run_tdd(cov, no_capture, pdb, partial, args)
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/__init__.py b/marvin-cli/marvin_cli/utils/__init__.py
new file mode 100644
index 0000000..77c2bbc
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/config.py b/marvin-cli/marvin_cli/utils/config.py
new file mode 100644
index 0000000..d68e063
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/config.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import copy
+import json
+import configparser
+
+def parse_ini(inipath, defaults=None):
+ if defaults is None:
+ defaults = {}
+
+ config_raw = configparser.ConfigParser()
+ config_raw.read(inipath)
+
+ config = copy.deepcopy(defaults)
+
+ for section in config_raw.sections():
+ # Firt pass
+ for key, value in config_raw.items(section):
+ key = '_'.join((section, key)).lower()
+ processed_value = value.format(**config)
+ config[key] = processed_value
+
+ # Second pass
+ for key, value in config.items():
+ processed_value = value.format(**config)
+ if ',' in processed_value:
+ processed_value = processed_value.split(',')
+ config[key] = processed_value
+
+ return config
+
+def read_cli_conf():
+ file_path = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
+ with open(file_path, 'r') as conf_file:
+ return json.load(conf_file)
+
+def generate_default_conf():
+ filepath = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
+ config = {
+ 'default_host': 'localhost',
+ 'editor': 'nano',
+ 'executor_url': 'https://s3.amazonaws.com/marvin-engine-executor/marvin-engine-executor-assembly-0.0.5.jar'
+ }
+ with open(filepath, 'w') as conf:
+ json.dump(config, conf, indent=4)
+
+ return config
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/docker.py b/marvin-cli/marvin_cli/utils/docker.py
new file mode 100644
index 0000000..269c3a9
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/docker.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from docker import DockerClient
+from docker.errors import ImageNotFound, NotFound
+from .misc import package_to_name, generate_engine_package
+from ..utils.log import get_logger
+
+logger = get_logger('docker')
+
+def _get_client(env=True, url=None):
+ if env:
+ return DockerClient.from_env()
+ return DockerClient(base_url=url)
+
+def _search_docker_image(name):
+ client = _get_client()
+ try:
+ client.images.get(name)
+ return True
+ except ImageNotFound:
+ return False
+
+def _search_docker_container(name):
+ client = _get_client()
+ try:
+ client.containers.get(name)
+ return True
+ except NotFound:
+ return False
+
+def search_engine_container(engine_package):
+ container_name = "marvin-cont-" + package_to_name(engine_package)
+ return _search_docker_container(container_name)
+
+def search_engine_images(engine_package):
+ image_name = "marvin-" + package_to_name(engine_package)
+ return _search_docker_image(image_name)
+
+def create_engine_image(engine_package):
+ logger.info("Creating engine docker image ...")
+ dockerfile_path = os.path.join(os.getcwd(), "docker",
+ "develop", "daemon")
+ generate_engine_package(engine_package)
+ client = _get_client()
+ client.images.build(
+ path=dockerfile_path,
+ tag="marvin-" + package_to_name(engine_package)
+ )
+ logger.info("Creating engine docker image ... Done!")
+
+def create_daemon_container(engine_package, engine_name):
+ logger.info("Creating engine docker container ...")
+ client = _get_client()
+ _engine_path = os.path.join(os.environ['MARVIN_HOME'], engine_name)
+ client.containers.run(
+ image="marvin-" + engine_name,
+ name="marvin-cont-" + engine_name,
+ volumes={
+ _engine_path: {
+ 'bind': '/opt/marvin/engine',
+ 'mode': 'rw'
+ },
+ os.environ['MARVIN_DATA_PATH']: {
+ 'bind': '/opt/marvin/data',
+ 'mode': 'rw'
+ },
+ os.environ['MARVIN_LOG']: {
+ 'bind': '/opt/marvin/log',
+ 'mode': 'rw'
+ }
+ },
+ ports={
+ '50051/tcp': 50051,
+ '50052/tcp': 50052,
+ '50053/tcp': 50053,
+ '50054/tcp': 50054,
+ '50055/tcp': 50055,
+ '50056/tcp': 50056,
+ '50057/tcp': 50057,
+ '8888/tcp': 8888
+ },
+ detach=True
+ )
+
+ logger.info("Creating engine docker container ... Done!")
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/git.py b/marvin-cli/marvin_cli/utils/git.py
new file mode 100644
index 0000000..6dec396
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/git.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+def git_init(path):
+ os.chdir(path)
+ os.system(' '.join(['git', 'init', '.']))
+
+def bump_version(part, verbose, dryrun):
+ command = ['bump2version', part]
+ if verbose:
+ command.append('--verbose')
+ if dryrun:
+ command.append('--dry-run')
+ os.system(' '.join(command))
+
diff --git a/marvin-cli/marvin_cli/utils/log.py b/marvin-cli/marvin_cli/utils/log.py
new file mode 100644
index 0000000..2417977
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/log.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Custom logging module.
+
+This module is responsible to manage log messages and log file.
+"""
+
+import os
+import os.path
+import logging
+
+DEFAULT_LOG_LEVEL = logging.INFO
+
+if os.environ.get('MARVIN_LOG') is None:
+ DEFAULT_LOG_DIR = '/tmp'
+else:
+ DEFAULT_LOG_DIR = os.environ['MARVIN_LOG']
+
+
+class Logger(logging.getLoggerClass()):
+ """Custom logger class.
+
+ Use this class to customize the logger behavior or to intercept the
+ messages.
+ """
+ # def error(self, msg, *args, **kwargs):
+ # # Add here code to intercept the project error messages
+ # super(Logger, self).error(msg, *args, **kwargs)
+
+ # def critical(self, msg, *args, **kwargs):
+ # # Add here code to intercept the project critical messages
+ # super(Logger, self).critical(msg, *args, **kwargs)
+
+
+logging.setLoggerClass(Logger)
+
+
+def get_logger(name, namespace='marvin_ai',
+ log_level=DEFAULT_LOG_LEVEL, log_dir=DEFAULT_LOG_DIR):
+ """Build a logger that outputs to a file and to the console,"""
+
+ log_level = (os.getenv('{}_LOG_LEVEL'.format(namespace.upper())) or
+ os.getenv('LOG_LEVEL', log_level))
+ log_dir = (os.getenv('{}_LOG_DIR'.format(namespace.upper())) or
+ os.getenv('LOG_DIR', log_dir))
+
+ logger = logging.getLogger('{}.{}'.format(namespace, name))
+ logger.setLevel(log_level)
+
+ formatter = logging.Formatter(
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ # Create a console stream handler
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(log_level)
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+ try:
+ log_path = os.path.abspath(log_dir)
+ log_filename = '{name}.{pid}.log'.format(
+ name=namespace, pid=os.getpid())
+
+ file_path = str(os.path.join(log_path, log_filename))
+
+ if not os.path.exists(log_path):
+ os.makedirs(log_path)
+
+ # Create a file handler
+ file_handler = logging.FileHandler(file_path)
+ file_handler.setLevel(log_level)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ except OSError as e:
+ logger.error('Could not create log file {file}: {error}'.format(
+ file=file_path, error=e.strerror))
+
+ return logger
\ No newline at end of file
diff --git a/marvin-cli/marvin_cli/utils/misc.py b/marvin-cli/marvin_cli/utils/misc.py
new file mode 100644
index 0000000..ad40429
--- /dev/null
+++ b/marvin-cli/marvin_cli/utils/misc.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import tarfile
+import wget
+
+def make_tarfile(output_filename, source_dir):
+ with tarfile.open(output_filename, "w:gz") as tar:
+ tar.add(source_dir, arcname=os.path.basename(source_dir))
+
+def package_to_name(package):
+ #remove marvin_ substring
+ return package[len("marvin_"):]
+
+def generate_engine_package(package):
+ subprocess.run(['python', 'setup.py', 'sdist'])
+ filename = package + "-" + get_version(package) + ".tar.gz"
+ origin = os.path.join(os.getcwd(), "dist", filename)
+ dest = os.path.join(os.getcwd(), "docker", "develop" ,"daemon", filename)
+ os.rename(origin, dest)
+
+def get_version(package):
+ with open(os.path.join(os.getcwd(), package ,"VERSION"), 'rb') as f:
+ version = f.read().decode('ascii').strip()
+ return version
+
+def package_folder(input, output):
+ with tarfile.open(output, "w:gz") as tar:
+ tar.add(input, arcname=os.path.basename(input))
+
+def extract_folder(input, output):
+ tf = tarfile.open(input)
+ tf.extractall(output)
+
+def call_logs(package):
+ container_name = 'marvin-cont-' + package_to_name(package)
+ subprocess.Popen(['xterm', '-e', 'docker', 'logs', '-f', container_name])
+
+def get_executor_path_or_download(executor_url):
+ #get filename from url
+ _executor_name = executor_url.split('/').pop(-1)
+
+ executor_path = os.path.join(os.environ['MARVIN_DATA_PATH'], _executor_name)
+
+ if not os.path.exists(executor_path):
+ wget(executor_url, out=executor_path)
+
+ return executor_path
\ No newline at end of file
diff --git a/marvin-cli/pytest.ini b/marvin-cli/pytest.ini
new file mode 100644
index 0000000..dbe65e1
--- /dev/null
+++ b/marvin-cli/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+minversion = 2.0
+norecursedirs = .git .tox .eggs .cache *.egg build dist tmp* marvin_cli/management
+python_files = test*.py
\ No newline at end of file
diff --git a/marvin-cli/setup.py b/marvin-cli/setup.py
new file mode 100644
index 0000000..b8dd8fa
--- /dev/null
+++ b/marvin-cli/setup.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os.path
+import os
+import sys
+
+from setuptools import setup, find_packages
+from setuptools.command.test import test as TestCommand
+
+# Package basic info
+PACKAGE_NAME = 'marvin_cli'
+PACKAGE_DESCRIPTION = 'Apache Marvin-AI CLI'
+
+URL = ''
+
+AUTHOR_NAME = 'Apache Marvin-AI Community'
+AUTHOR_EMAIL = 'dev@marvin.apache.org'
+
+PYTHON_2 = False
+PYTHON_3 = True
+
+# Project status
+# (should be 'planning', 'pre-alpha', 'alpha', 'beta', 'stable', 'mature' or 'inactive').
+STATUS = 'planning'
+
+# Project topic
+# See https://pypi.python.org/pypi?%3Aaction=list_classifiers for a list
+TOPIC = 'Topic :: Software Development :: Libraries :: Python Modules',
+
+# External dependencies
+# More info https://pythonhosted.org/setuptools/setuptools.html#declaring-dependencies
+REQUIREMENTS_EXTERNAL = [
+ 'docker',
+ 'configparser',
+ 'bump2version',
+ 'wget',
+ 'grpcio',
+ 'grpcio-tools',
+ 'click',
+ 'cookiecutter'
+]
+
+# This is normally an empty list
+DEPENDENCY_LINKS_EXTERNAL = []
+# script to be used
+SCRIPTS = ['bin/marvin']
+
+
+def _get_version():
+ """Return the project version from VERSION file."""
+ with open(os.path.join(os.path.dirname(__file__), PACKAGE_NAME, 'VERSION'), 'rb') as f:
+ version = f.read().decode('ascii').strip()
+ return version
+
+
+DEVELOPMENT_STATUS = {
+ 'planning': '1 - Planning',
+ 'pre-alpha': '2 - Pre-Alpha',
+ 'alpha': 'Alpha',
+ 'beta': '4 - Beta',
+ 'stable': '5 - Production/Stable',
+ 'mature': '6 - Mature',
+ 'inactive': '7 - Inactive',
+}
+
+CLASSIFIERS = ['Development Status :: {}'.format(DEVELOPMENT_STATUS[STATUS])]
+if PYTHON_2:
+ CLASSIFIERS += [
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ ]
+if PYTHON_3:
+ CLASSIFIERS += [
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.6',
+ ]
+
+setup(
+ name=PACKAGE_NAME,
+ version=_get_version(),
+ url=URL,
+ description=PACKAGE_DESCRIPTION,
+ long_description=open(os.path.join(
+ os.path.dirname(__file__), 'README.md')).read(),
+ author=AUTHOR_NAME,
+ maintainer=AUTHOR_NAME,
+ maintainer_email=AUTHOR_EMAIL,
+ packages=find_packages(exclude=('tests', 'tests.*')),
+ include_package_data=True,
+ zip_safe=False,
+ classifiers=CLASSIFIERS,
+ install_requires=REQUIREMENTS_EXTERNAL,
+ dependency_links=DEPENDENCY_LINKS_EXTERNAL,
+ scripts=SCRIPTS,
+)
diff --git a/marvin-cli/tests/communication/test_remote_calls.py b/marvin-cli/tests/communication/test_remote_calls.py
new file mode 100644
index 0000000..d3564dd
--- /dev/null
+++ b/marvin-cli/tests/communication/test_remote_calls.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.communication.remote_calls import RemoteCalls
+
+mocked_host = '0.0.0.0'
+mocked_port = 0
+
+mocked_name = 'GRPC'
+mocked_params = {
+ 'mocked': 'params'
+}
+mocked_args = 'mocked_args'
+
+@mock.patch('marvin_cli.communication.remote_calls.grpc.insecure_channel')
+def test_RemoteCall_init(channel_mocked):
+ RemoteCalls(mocked_host, mocked_port)
+ channel_mocked.assert_called_with('0.0.0.0:0')
+
+@mock.patch('marvin_cli.communication.remote_calls.daemon_pb2_grpc.CommandCallStub')
+def test_call_command(stub_mocked):
+ stub = stub_mocked.return_value
+ rc = RemoteCalls()
+ rc.call_command(mocked_name, mocked_params)
+ stub.callCommand.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.daemon_pb2_grpc.CommandCallStub')
+def test_stop_command(stub_mocked):
+ stub = stub_mocked.return_value
+ rc = RemoteCalls()
+ rc.stop_command(mocked_name)
+ stub.stopCommand.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_grpc(call_mocked):
+ rc = RemoteCalls()
+ rc.run_grpc('all', None, None)
+ call_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.stop_command')
+def test_stop_grpc(stop_mocked):
+ rc = RemoteCalls()
+ rc.stop_grpc()
+ stop_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_dryrun(call_mocked):
+ rc = RemoteCalls()
+ rc.run_dryrun('all', True)
+ call_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_notebook(call_mocked):
+ rc = RemoteCalls()
+ rc.run_notebook(True, True)
+ call_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_lab(call_mocked):
+ rc = RemoteCalls()
+ rc.run_lab(True, True)
+ call_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_test(call_mocked):
+ rc = RemoteCalls()
+ rc.run_test(True, True, True, mocked_args)
+ call_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_tox(call_mocked):
+ rc = RemoteCalls()
+ rc.run_tox(mocked_args)
+ call_mocked.assert_called()
+
+@mock.patch('marvin_cli.communication.remote_calls.RemoteCalls.call_command')
+def test_run_tdd(call_mocked):
+ rc = RemoteCalls()
+ rc.run_tdd(True, True, True, True, mocked_args)
+ call_mocked.assert_called()
diff --git a/marvin-cli/tests/management/test_edit.py b/marvin-cli/tests/management/test_edit.py
new file mode 100644
index 0000000..00eba18
--- /dev/null
+++ b/marvin-cli/tests/management/test_edit.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import click
+import click.testing
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.management.edit import config
+from marvin_cli.management.edit import metadata
+mocked_obj = {
+ 'editor': 'mocked'
+}
+
+def test_config():
+ ctx = click.Context(click.Command('edit-config'), obj=mocked_obj)
+ with ctx:
+ runner = click.testing.CliRunner()
+ runner.invoke(config)
+
+def test_metadata():
+ ctx = click.Context(click.Command('edit-metadata'), obj=mocked_obj)
+ with ctx:
+ runner = click.testing.CliRunner()
+ runner.invoke(metadata)
+
+
+
diff --git a/marvin-cli/tests/management/test_engine.py b/marvin-cli/tests/management/test_engine.py
new file mode 100644
index 0000000..d59222f
--- /dev/null
+++ b/marvin-cli/tests/management/test_engine.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import click
+import click.testing
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.management.engine import generate
+
+@mock.patch('marvin_cli.management.engine.cookiecutter')
+def test_generate(cookie_mocked):
+ args = ['-n', 'mocked', '-d', 'mocked', '-m',
+ 'mocked', '-e', 'mocked']
+ runner = click.testing.CliRunner()
+ result = runner.invoke(generate, args)
+ cookie_mocked.assert_called()
diff --git a/marvin-cli/tests/utils/test_config.py b/marvin-cli/tests/utils/test_config.py
new file mode 100644
index 0000000..b627b85
--- /dev/null
+++ b/marvin-cli/tests/utils/test_config.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.utils.config import parse_ini
+from marvin_cli.utils.config import read_cli_conf
+from marvin_cli.utils.config import generate_default_conf
+
+mocked_path = '/path/to/nowhere'
+
+@mock.patch('marvin_cli.utils.config.configparser.ConfigParser.read')
+def test_parse_ini(read_mocked):
+ parse_ini(mocked_path)
+ read_mocked.assert_called_with(mocked_path)
+
+@mock.patch('marvin_cli.utils.config.json.load')
+@mock.patch('marvin_cli.utils.config.open')
+def test_read_cli_conf(open_mocked, load_mocked):
+ _conf_path = os.path.join(os.environ['MARVIN_DATA_PATH'], '.conf', 'cli_conf.json')
+ read_cli_conf()
+ open_mocked.assert_called_with(_conf_path, 'r')
+ load_mocked.assert_called()
+
+@mock.patch('marvin_cli.utils.config.json.dump')
+def test_generate_default_conf(dump_mocked):
+ generate_default_conf()
+ dump_mocked.assert_called()
\ No newline at end of file
diff --git a/marvin-cli/tests/utils/test_docker.py b/marvin-cli/tests/utils/test_docker.py
new file mode 100644
index 0000000..5360883
--- /dev/null
+++ b/marvin-cli/tests/utils/test_docker.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.utils.docker import _get_client
+from marvin_cli.utils.docker import _search_docker_image
+from marvin_cli.utils.docker import _search_docker_container
+from marvin_cli.utils.docker import search_engine_container
+from marvin_cli.utils.docker import search_engine_images
+from marvin_cli.utils.docker import create_engine_image
+from marvin_cli.utils.docker import create_daemon_container
+
+@mock.patch('marvin_cli.utils.docker.DockerClient.from_env')
+def test_get_client(env_mocked):
+ _get_client()
+ env_mocked.assert_called()
+
+@mock.patch('marvin_cli.utils.docker.DockerClient')
+def test_get_client_remote(client_mocked):
+ _get_client(env=False, url='docker://mocked_url')
+ client_mocked.assert_called_with(base_url='docker://mocked_url')
+
+def test_search_docker_image():
+ assert not _search_docker_image('mocked_name')
+
+def test_search_docker_container():
+ assert not _search_docker_container('mocked_name')
+
+@mock.patch('marvin_cli.utils.docker._search_docker_container')
+def test_search_engine_container(search_mocked):
+ search_engine_container('marvin_mocked')
+ search_mocked.assert_called_with('marvin-cont-mocked')
+
+@mock.patch('marvin_cli.utils.docker._search_docker_image')
+def test_search_engine_images(search_mocked):
+ search_engine_images('marvin_mocked')
+ search_mocked.assert_called_with('marvin-mocked')
+
+@mock.patch('marvin_cli.utils.docker._get_client')
+@mock.patch('marvin_cli.utils.docker.generate_engine_package')
+def test_create_engine_image(generate_mocked, client_mocked):
+ create_engine_image('marvin_mocked')
+ client_mocked.assert_called()
+ generate_mocked.assert_called_with('marvin_mocked')
+
+@mock.patch('marvin_cli.utils.docker._get_client')
+def test_create_daemon_container(client_mocked):
+ create_daemon_container('marvin_mocked', 'mocked')
+ client_mocked.assert_called()
\ No newline at end of file
diff --git a/marvin-cli/tests/utils/test_git.py b/marvin-cli/tests/utils/test_git.py
new file mode 100644
index 0000000..a20ea5a
--- /dev/null
+++ b/marvin-cli/tests/utils/test_git.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.utils.git import git_init
+from marvin_cli.utils.git import bump_version
+
+mocked_path = '/path/to/nowhere'
+
+@mock.patch('marvin_cli.utils.git.os.system')
+@mock.patch('marvin_cli.utils.git.os.chdir')
+def test_git_init(chdir_mocked, system_mocked):
+ git_init(mocked_path)
+ system_mocked.assert_called_with('git init .')
+ chdir_mocked.assert_called()
+
+@mock.patch('marvin_cli.utils.git.os.system')
+def test_bumpversion(system_mocked):
+ bump_version('patch', True, True)
+ system_mocked.assert_called_with('bump2version patch --verbose --dry-run')
\ No newline at end of file
diff --git a/marvin-cli/tests/utils/test_misc.py b/marvin-cli/tests/utils/test_misc.py
new file mode 100644
index 0000000..c10ca59
--- /dev/null
+++ b/marvin-cli/tests/utils/test_misc.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright [2020] [Apache Software Foundation]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+try:
+ import mock
+except ImportError:
+ import unittest.mock as mock
+
+from marvin_cli.utils.misc import make_tarfile
+from marvin_cli.utils.misc import package_to_name
+from marvin_cli.utils.misc import get_version
+from marvin_cli.utils.misc import package_folder
+from marvin_cli.utils.misc import extract_folder
+from marvin_cli.utils.misc import call_logs
+from marvin_cli.utils.misc import get_executor_path_or_download
+
+mocked_package = 'marvin_mocked'
+mocked_path = '/path/to/nowhere'
+mocked_url = 'https://some.url/marvin.jar'
+
+@mock.patch('marvin_cli.utils.misc.tarfile.open')
+def test_make_tarfile(open_mocked):
+ make_tarfile(mocked_path, mocked_path)
+ open_mocked.assert_called_with(mocked_path, 'w:gz')
+
+def test_package_to_name():
+ assert package_to_name(mocked_package) == 'mocked'
+
+@mock.patch('marvin_cli.utils.misc.os.path.join')
+def test_get_version(join_mocked):
+ get_version(mocked_package)
+ join_mocked.assert_called_with(os.getcwd(), mocked_package, 'VERSION')
+
+@mock.patch('marvin_cli.utils.misc.tarfile.open')
+def test_package_folder(open_mocked):
+ package_folder(mocked_path, mocked_path)
+ open_mocked.assert_called_with(mocked_path, 'w:gz')
+
+@mock.patch('marvin_cli.utils.misc.tarfile.open')
+def test_extract_folder(open_mocked):
+ extract_folder(mocked_path, mocked_path)
+ open_mocked.assert_called_with(mocked_path)
+
+@mock.patch('marvin_cli.utils.misc.subprocess.Popen')
+def test_call_logs(popen_mocked):
+ call_logs(mocked_package)
+ popen_mocked.assert_called_with(['xterm', '-e', 'docker', 'logs', '-f', 'marvin-cont-mocked'])
+
+@mock.patch('marvin_cli.utils.misc.wget')
+def test_get_executor_path_or_download(wget_mocked):
+ get_executor_path_or_download(mocked_url)
+ wget_mocked.assert_called_with(mocked_url, out=os.path.join(os.environ['MARVIN_DATA_PATH'], 'marvin.jar'))
\ No newline at end of file
diff --git a/marvin-cli/tox.ini b/marvin-cli/tox.ini
new file mode 100644
index 0000000..c059be8
--- /dev/null
+++ b/marvin-cli/tox.ini
@@ -0,0 +1,9 @@
+[tox]
+envlist = py36
+
+[testenv]
+deps=pytest
+ pytest-cov
+ mock
+commands=py.test --cov={envsitepackagesdir}/marvin_cli --cov-report html --cov-report xml {posargs}
+passenv=SPARK_HOME MARVIN_HOME MARVIN_DATA_PATH MARVIN_LOG
[incubator-marvin] 03/04: Change build folder to build daemon
docker image
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit 7f708a23a23c6ce631f5fe8674042a5026f0710e
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Aug 26 17:41:27 2020 -0300
Change build folder to build daemon docker image
---
Makefile | 36 +++++++++---------------------
build/daemon/Dockerfile | 8 +++++++
build/marvin-engine-executor-assembly.jar | Bin 94184375 -> 0 bytes
build/python-toolbox.tgz | Bin 693248 -> 0 bytes
4 files changed, 19 insertions(+), 25 deletions(-)
diff --git a/Makefile b/Makefile
index f0a548c..67333f4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,36 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-.PHONY: help docker-base docker-python docker-r toolbox engine-server
+.PHONY: help python-daemon-base engine-server python-daemon
help:
- @echo " docker-base"
- @echo " Builds the base docker image with common dependecies among languages."
- @echo " docker-python"
+ @echo " python-daemon-base"
@echo " Builds a docker image that can run engines in Python."
- @echo " docker-r"
- @echo " Builds a docker image that can run engines in R."
- @echo " toolbox"
- @echo " Builds the toolbox and make it available for be included into docker images."
+ @echo " python-daemon"
+ @echo " Builds the python-daemon and make it available for be included into docker images."
@echo " engine-server"
@echo " Builds a jar with the engine interpreter server."
-docker-base:
- docker build -t marvin-base -f engine-server/build/Dockerfile engine-server/build
-
-docker-python:
- $(MAKE) engine-server
- $(MAKE) toolbox
- docker build -t marvin-python -f engine-server/build/docker-python/Dockerfile engine-server/build
-
-docker-r:
- $(MAKE) engine-server
- $(MAKE) toolbox
- docker build -t marvin-r -f engine-server/build/docker-r/Dockerfile engine-server/build
+python-daemon-base:
+ $(MAKE) python-daemon
+ docker build -t marvin-daemon:python -f build/daemon/Dockerfile build/daemon
engine-server:
- cd engine-server && $(MAKE) package
- mv engine-server/target/scala-2.12/marvin-engine-server-assembly-*.jar engine-server/build/marvin-engine-server-assembly.jar
+ cd engine-executor && $(MAKE) package
+ mv engine-executor/target/scala-2.12/marvin-engine-executor-assembly-*.jar build/executor/marvin-engine-executor-assembly.jar
-toolbox:
- tar -cf engine-server/build/python-toolbox.tgz python-toolbox
\ No newline at end of file
+python-daemon:
+ tar -zcvf build/daemon/python-daemon.tar.gz python-daemon
\ No newline at end of file
diff --git a/build/daemon/Dockerfile b/build/daemon/Dockerfile
new file mode 100644
index 0000000..1154b3d
--- /dev/null
+++ b/build/daemon/Dockerfile
@@ -0,0 +1,8 @@
+FROM python:3.7-buster
+
+ADD python-daemon.tar.gz .
+RUN /bin/bash -c "apt update && apt install libsasl2-dev graphviz -y" \
+&& /bin/bash -c "cd python-daemon && pip install ." \
+&& rm -rf python-daemon \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
\ No newline at end of file
diff --git a/build/marvin-engine-executor-assembly.jar b/build/marvin-engine-executor-assembly.jar
deleted file mode 100644
index ff8a8f4..0000000
Binary files a/build/marvin-engine-executor-assembly.jar and /dev/null differ
diff --git a/build/python-toolbox.tgz b/build/python-toolbox.tgz
deleted file mode 100644
index 86b60d1..0000000
Binary files a/build/python-toolbox.tgz and /dev/null differ
[incubator-marvin] 04/04: Add cli in CI
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichen pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-marvin.git
commit 716bd93efa4eb592da6e38543240e97b63dd6c5d
Author: cardosolucas <ca...@gmail.com>
AuthorDate: Wed Aug 26 17:44:19 2020 -0300
Add cli in CI
---
.travis.yml | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/.travis.yml b/.travis.yml
index 7c5feef..958de0b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,3 +61,32 @@ matrix:
script:
- tox
- codecov
+ # Python CLI - Linux
+ - language: python
+ os: linux
+ python:
+ - 3.6
+ before_install:
+ - travis_retry curl https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz -o ./spark-2.1.1-bin-hadoop2.6.tgz
+ - sudo tar -xf ./spark-2.1.1-bin-hadoop2.6.tgz
+ - cd marvin-cli
+ - mkdir -p marvin_data
+ - mkdir -p marvin_home
+ - mkdir -p marvin_log
+ - export MARVIN_LOG=./marvin_log
+ - export MARVIN_HOME=./marvin_home
+ - export MARVIN_DATA_PATH=./marvin_data
+ - export SPARK_HOME=../spark-2.1.1-bin-hadoop2.6
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libsasl2-dev python-pip graphviz -y ; fi
+ - travis_retry sudo pip install --upgrade pip
+ - travis_retry sudo pip install virtualenvwrapper --ignore-installed six
+ - source virtualenvwrapper.sh
+ install:
+ - travis_retry pip install codecov
+ - travis_retry pip install unidecode
+ - travis_retry pip install tox
+ - make install
+ - marvin
+ script:
+ - travis_wait 30 tox
+ - codecov